{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对上周学习目标的回顾\n",
    "1. 使用 requests-html 爬取并存取网页文字档，查找requests-html 中文文档\n",
    "2. 熟悉 xpath 语法丶xpath 节点\n",
    "3. 使用 xpath cheatsheet\n",
    "    * 在 Chrome Inspector 使用\n",
    "    * 在 requests-html (Python) 使用\n",
    "4. 简易使用 pd.DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 猎聘移动端(代码由老师提供，本人添加注释)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 基本模块\n",
    "import pandas as pd\n",
    "from requests_html import HTMLSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>职称</th>\n",
       "      <th>薪水</th>\n",
       "      <th>公司地点</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>时间</th>\n",
       "      <th>经验</th>\n",
       "      <th>链结</th>\n",
       "      <th>公司URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-23k·13薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>数果智能</td>\n",
       "      <td>10小时前</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926363353.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8837861/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州</td>\n",
       "      <td>如祺出行</td>\n",
       "      <td>昨天</td>\n",
       "      <td>3年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1927117161.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10094945/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-16k·12薪</td>\n",
       "      <td>广州-新市</td>\n",
       "      <td>广州伊羽霓贸易有限公司</td>\n",
       "      <td>昨天</td>\n",
       "      <td>2年以上 大专及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926600635.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9705250/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>18-33k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>卓越教育集团</td>\n",
       "      <td>昨天</td>\n",
       "      <td>5年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926567413.shtml</td>\n",
       "      <td>https://m.liepin.com/company/876848/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-15k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>大麦信息</td>\n",
       "      <td>昨天</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926420031.shtml</td>\n",
       "      <td>https://m.liepin.com/company/7870705/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-25k·12薪</td>\n",
       "      <td>广州-番禺区</td>\n",
       "      <td>广州探迹科技有限公司</td>\n",
       "      <td>昨天</td>\n",
       "      <td>5年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926292629.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8836020/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>6-8k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>嫚熙控股(广州)有限公司</td>\n",
       "      <td>昨天</td>\n",
       "      <td>3年以上 大专及以上</td>\n",
       "      <td>https://m.liepin.com/job/1925238817.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9660700/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>20-35k·15薪</td>\n",
       "      <td>北京,广州,上海</td>\n",
       "      <td>xx公司</td>\n",
       "      <td>2020-03-26</td>\n",
       "      <td>2年以上 学历不限</td>\n",
       "      <td>https://m.liepin.com/a/18569421.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-12k·13薪</td>\n",
       "      <td>广州-天河北</td>\n",
       "      <td>维他奶</td>\n",
       "      <td>昨天</td>\n",
       "      <td>1年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1924641017.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8428035/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州-天河北</td>\n",
       "      <td>租租车</td>\n",
       "      <td>昨天</td>\n",
       "      <td>1年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1921434973.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8082331/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-25k·13薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>国内游戏知名企业</td>\n",
       "      <td>昨天</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/19077453.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-25k·13薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>国内游戏知名企业</td>\n",
       "      <td>昨天</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/19077429.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-25k·13薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>某知名互联网公司</td>\n",
       "      <td>昨天</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/18784931.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州-林和</td>\n",
       "      <td>租租车</td>\n",
       "      <td>昨天</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1913960836.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8082331/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州</td>\n",
       "      <td>欢聚集团</td>\n",
       "      <td>前天</td>\n",
       "      <td>1年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1927186335.shtml</td>\n",
       "      <td>https://m.liepin.com/company/930104/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-12k·12薪</td>\n",
       "      <td>广州-东圃</td>\n",
       "      <td>广州粤达互娱科技有限公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1927025161.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10255845/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12-20k·12薪</td>\n",
       "      <td>广州-海珠区</td>\n",
       "      <td>广州滴普科技有限公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>5年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926885893.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10166945/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-15k·13薪</td>\n",
       "      <td>广州-五山</td>\n",
       "      <td>广州游爱网络技术有限公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926606033.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8037518/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-15k·13薪</td>\n",
       "      <td>广州-花都区</td>\n",
       "      <td>广州华骏电子商务有限责任公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>4年以上 大专及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926501261.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9932713/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>6-12k·12薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>金蝶医疗</td>\n",
       "      <td>前天</td>\n",
       "      <td>经验不限 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926109895.shtml</td>\n",
       "      <td>https://m.liepin.com/company/6780253/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-15k·12薪</td>\n",
       "      <td>广州-东风东</td>\n",
       "      <td>海南大圣娱乐有限公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>1年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1925949165.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9996995/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-14k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>顺丰速运有限公司区域共享中心</td>\n",
       "      <td>前天</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1925922047.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8880560/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-15k·12薪</td>\n",
       "      <td>广州-东湖</td>\n",
       "      <td>申迪</td>\n",
       "      <td>前天</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1925414753.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8067064/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-10k·13薪</td>\n",
       "      <td>广州-龙凤</td>\n",
       "      <td>厦门洛基电子商务有限公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>3年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1924656343.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10178781/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-30k·15薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>发展型互联网保险第三方经纪平台</td>\n",
       "      <td>前天</td>\n",
       "      <td>5年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/19622953.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>13-18k·14薪</td>\n",
       "      <td>广州-海珠区</td>\n",
       "      <td>蜗牛保险</td>\n",
       "      <td>前天</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/19618915.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-15k·12薪</td>\n",
       "      <td>广州-建设</td>\n",
       "      <td>广州市时代胜誉投资有限公司</td>\n",
       "      <td>前天</td>\n",
       "      <td>3年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1911819089.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9126093/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州</td>\n",
       "      <td>壹链盟生态科技有限公司</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>2年以上 硕士及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926968307.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9903267/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-15k·12薪</td>\n",
       "      <td>广州-海珠区</td>\n",
       "      <td>广州真义电子商务有限公司</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1925915631.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10176153/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12-20k·12薪</td>\n",
       "      <td>广州-增城区</td>\n",
       "      <td>广州市增城区城乡规划与测绘地理信息研究院</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>经验不限 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1922192733.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10141823/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12-20k·13薪</td>\n",
       "      <td>广州-车陂</td>\n",
       "      <td>乐其电商</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1919988491.shtml</td>\n",
       "      <td>https://m.liepin.com/company/1543625/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-15k·12薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>广州天维信息技术股份有限公司</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1918854685.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9656308/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>20-40k·12薪</td>\n",
       "      <td>广州-番禺区</td>\n",
       "      <td>广州虎牙信息科技有限公司</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1917412855.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9287281/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>20-40k·13薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州虎牙信息科技有限公司</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1917411127.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9287281/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-15k·12薪</td>\n",
       "      <td>广州-五山</td>\n",
       "      <td>上海联蔚信息科技有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>5年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1927112101.shtml</td>\n",
       "      <td>https://m.liepin.com/company/3547412/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-20k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州卓铸网络科技有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>3年以上 大专及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926949151.shtml</td>\n",
       "      <td>https://m.liepin.com/company/12179843/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-12k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州市新娱加娱乐传媒文化有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>经验不限 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1925880227.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8976402/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-15k·12薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>广州云趣信息科技有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1922622715.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9638401/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-15k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州市奔流电力科技有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>经验不限 学历不限</td>\n",
       "      <td>https://m.liepin.com/job/1920244061.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9226181/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>5-8k·12薪</td>\n",
       "      <td>广州-同德</td>\n",
       "      <td>福建省三福百货有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>5年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/199674161.shtml</td>\n",
       "      <td>https://m.liepin.com/company/7892336/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>4-6k·13薪</td>\n",
       "      <td>广州-同德</td>\n",
       "      <td>福建省三福百货有限公司</td>\n",
       "      <td>2020-03-31</td>\n",
       "      <td>1年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/198635044.shtml</td>\n",
       "      <td>https://m.liepin.com/company/7892336/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州-黄埔区</td>\n",
       "      <td>京东物流</td>\n",
       "      <td>2020-03-30</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1926948641.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9770323/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-12k·13薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州梵之容化妆品有限公司</td>\n",
       "      <td>2020-03-30</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926195541.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9918971/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12-16k·14薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>某互联网保险公司</td>\n",
       "      <td>2020-03-30</td>\n",
       "      <td>3年以上 学历不限</td>\n",
       "      <td>https://m.liepin.com/a/19495749.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>30-50k·12薪</td>\n",
       "      <td>广州,深圳,佛山</td>\n",
       "      <td>JLW</td>\n",
       "      <td>2020-03-30</td>\n",
       "      <td>3年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/19347549.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>20-40k·12薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>广州某网络科技有限公司</td>\n",
       "      <td>2020-03-30</td>\n",
       "      <td>1年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/a/19063297.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>8-20k·12薪</td>\n",
       "      <td>广州-东风</td>\n",
       "      <td>新东方教育科技集团有限公司</td>\n",
       "      <td>2020-03-27</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926946115.shtml</td>\n",
       "      <td>https://m.liepin.com/company/2485876/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12-18k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州艾美网络科技有限公司</td>\n",
       "      <td>2020-03-27</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1924672993.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8801986/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州</td>\n",
       "      <td>如祺出行</td>\n",
       "      <td>2020-03-26</td>\n",
       "      <td>2年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926903167.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10094945/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-22k·12薪</td>\n",
       "      <td>广州-荔湾区</td>\n",
       "      <td>一手服装批发APP</td>\n",
       "      <td>2020-03-26</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1923664395.shtml</td>\n",
       "      <td>https://m.liepin.com/company/8955475/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-25k·12薪</td>\n",
       "      <td>广州-海珠区</td>\n",
       "      <td>北京路客互联网科技有限公司</td>\n",
       "      <td>2020-03-25</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1924692443.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9284656/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>5-10k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>时代大数据</td>\n",
       "      <td>2020-03-25</td>\n",
       "      <td>1年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1920605757.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9382271/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>30-50k·14薪</td>\n",
       "      <td>北京-东城区,上海-黄浦区,广州-黄埔区</td>\n",
       "      <td>国内某大型互联网公司</td>\n",
       "      <td>2020-03-25</td>\n",
       "      <td>1年以上 学历不限</td>\n",
       "      <td>https://m.liepin.com/a/19376743.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>5-7k·12薪</td>\n",
       "      <td>广州-天河区</td>\n",
       "      <td>广州民间金融街征信有限公司</td>\n",
       "      <td>2020-03-23</td>\n",
       "      <td>经验不限 大专及以上</td>\n",
       "      <td>https://m.liepin.com/job/1922291239.shtml</td>\n",
       "      <td>https://m.liepin.com/company/10056971/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>12-20k·12薪</td>\n",
       "      <td>广州-东圃</td>\n",
       "      <td>广州赛泽科技有限公司</td>\n",
       "      <td>2020-03-23</td>\n",
       "      <td>2年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1921893265.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9411137/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>面议</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州虎牙信息科技有限公司</td>\n",
       "      <td>2020-03-23</td>\n",
       "      <td>经验不限 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1917586595.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9287281/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-15k·13薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>亚信科技</td>\n",
       "      <td>2020-03-20</td>\n",
       "      <td>3年以上 统招本科</td>\n",
       "      <td>https://m.liepin.com/job/1926654193.shtml</td>\n",
       "      <td>https://m.liepin.com/company/2399664/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>10-25k·12薪</td>\n",
       "      <td>广州-赤岗</td>\n",
       "      <td>北京路客互联网科技有限公司</td>\n",
       "      <td>2020-03-19</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1924683829.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9284656/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>15-20k·13薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>广州花生日记网络科技有限公司</td>\n",
       "      <td>2020-03-19</td>\n",
       "      <td>3年以上 本科及以上</td>\n",
       "      <td>https://m.liepin.com/job/1923167241.shtml</td>\n",
       "      <td>https://m.liepin.com/company/9680336/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>数据分析师</td>\n",
       "      <td>20-30k·15薪</td>\n",
       "      <td>北京,广州,深圳</td>\n",
       "      <td>某知名外资公司</td>\n",
       "      <td>2020-03-19</td>\n",
       "      <td>2年以上 大专及以上</td>\n",
       "      <td>https://m.liepin.com/a/19269411.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        职称          薪水                  公司地点                  公司名称  \\\n",
       "0   数据分析师   15-23k·13薪                广州-天河区                  数果智能   \n",
       "1   数据分析师           面议                    广州                  如祺出行   \n",
       "2   数据分析师   10-16k·12薪                 广州-新市           广州伊羽霓贸易有限公司   \n",
       "3   数据分析师   18-33k·12薪                    广州                卓越教育集团   \n",
       "4   数据分析师    8-15k·12薪                    广州                  大麦信息   \n",
       "5   数据分析师   15-25k·12薪                广州-番禺区            广州探迹科技有限公司   \n",
       "6   数据分析师     6-8k·12薪                    广州          嫚熙控股(广州)有限公司   \n",
       "7   数据分析师   20-35k·15薪              北京,广州,上海                  xx公司   \n",
       "8   数据分析师    8-12k·13薪                广州-天河北                   维他奶   \n",
       "9   数据分析师           面议                广州-天河北                   租租车   \n",
       "10  数据分析师   15-25k·13薪                广州-天河区              国内游戏知名企业   \n",
       "11  数据分析师   15-25k·13薪                广州-天河区              国内游戏知名企业   \n",
       "12  数据分析师   15-25k·13薪                广州-天河区              某知名互联网公司   \n",
       "13  数据分析师           面议                 广州-林和                   租租车   \n",
       "14  数据分析师           面议                    广州                  欢聚集团   \n",
       "15  数据分析师    8-12k·12薪                 广州-东圃          广州粤达互娱科技有限公司   \n",
       "16  数据分析师   12-20k·12薪                广州-海珠区            广州滴普科技有限公司   \n",
       "17  数据分析师   10-15k·13薪                 广州-五山          广州游爱网络技术有限公司   \n",
       "18  数据分析师   10-15k·13薪                广州-花都区        广州华骏电子商务有限责任公司   \n",
       "19  数据分析师    6-12k·12薪                广州-天河区                  金蝶医疗   \n",
       "20  数据分析师    8-15k·12薪                广州-东风东            海南大圣娱乐有限公司   \n",
       "21  数据分析师    8-14k·12薪                    广州        顺丰速运有限公司区域共享中心   \n",
       "22  数据分析师    8-15k·12薪                 广州-东湖                    申迪   \n",
       "23  数据分析师    8-10k·13薪                 广州-龙凤          厦门洛基电子商务有限公司   \n",
       "24  数据分析师   15-30k·15薪                    广州       发展型互联网保险第三方经纪平台   \n",
       "25  数据分析师   13-18k·14薪                广州-海珠区                  蜗牛保险   \n",
       "26  数据分析师    8-15k·12薪                 广州-建设         广州市时代胜誉投资有限公司   \n",
       "27  数据分析师           面议                    广州           壹链盟生态科技有限公司   \n",
       "28  数据分析师    8-15k·12薪                广州-海珠区          广州真义电子商务有限公司   \n",
       "29  数据分析师   12-20k·12薪                广州-增城区  广州市增城区城乡规划与测绘地理信息研究院   \n",
       "30  数据分析师   12-20k·13薪                 广州-车陂                  乐其电商   \n",
       "31  数据分析师   10-15k·12薪                广州-天河区        广州天维信息技术股份有限公司   \n",
       "32  数据分析师   20-40k·12薪                广州-番禺区          广州虎牙信息科技有限公司   \n",
       "33  数据分析师   20-40k·13薪                    广州          广州虎牙信息科技有限公司   \n",
       "34  数据分析师   10-15k·12薪                 广州-五山          上海联蔚信息科技有限公司   \n",
       "35  数据分析师   10-20k·12薪                    广州          广州卓铸网络科技有限公司   \n",
       "36  数据分析师    8-12k·12薪                    广州      广州市新娱加娱乐传媒文化有限公司   \n",
       "37  数据分析师   10-15k·12薪                广州-天河区          广州云趣信息科技有限公司   \n",
       "38  数据分析师    8-15k·12薪                    广州         广州市奔流电力科技有限公司   \n",
       "39  数据分析师     5-8k·12薪                 广州-同德           福建省三福百货有限公司   \n",
       "40  数据分析师     4-6k·13薪                 广州-同德           福建省三福百货有限公司   \n",
       "41  数据分析师           面议                广州-黄埔区                  京东物流   \n",
       "42  数据分析师    8-12k·13薪                    广州          广州梵之容化妆品有限公司   \n",
       "43  数据分析师   12-16k·14薪                    广州              某互联网保险公司   \n",
       "44  数据分析师   30-50k·12薪              广州,深圳,佛山                   JLW   \n",
       "45  数据分析师   20-40k·12薪                广州-天河区           广州某网络科技有限公司   \n",
       "46  数据分析师    8-20k·12薪                 广州-东风         新东方教育科技集团有限公司   \n",
       "47  数据分析师   12-18k·12薪                    广州          广州艾美网络科技有限公司   \n",
       "48  数据分析师           面议                    广州                  如祺出行   \n",
       "49  数据分析师   15-22k·12薪                广州-荔湾区             一手服装批发APP   \n",
       "50  数据分析师   10-25k·12薪                广州-海珠区         北京路客互联网科技有限公司   \n",
       "51  数据分析师    5-10k·12薪                    广州                 时代大数据   \n",
       "52  数据分析师   30-50k·14薪  北京-东城区,上海-黄浦区,广州-黄埔区            国内某大型互联网公司   \n",
       "53  数据分析师     5-7k·12薪                广州-天河区         广州民间金融街征信有限公司   \n",
       "54  数据分析师   12-20k·12薪                 广州-东圃            广州赛泽科技有限公司   \n",
       "55  数据分析师           面议                    广州          广州虎牙信息科技有限公司   \n",
       "56  数据分析师   10-15k·13薪                    广州                  亚信科技   \n",
       "57  数据分析师   10-25k·12薪                 广州-赤岗         北京路客互联网科技有限公司   \n",
       "58  数据分析师   15-20k·13薪                    广州        广州花生日记网络科技有限公司   \n",
       "59  数据分析师   20-30k·15薪              北京,广州,深圳               某知名外资公司   \n",
       "\n",
       "            时间          经验                                         链结  \\\n",
       "0        10小时前  3年以上 本科及以上  https://m.liepin.com/job/1926363353.shtml   \n",
       "1           昨天   3年以上 统招本科  https://m.liepin.com/job/1927117161.shtml   \n",
       "2           昨天  2年以上 大专及以上  https://m.liepin.com/job/1926600635.shtml   \n",
       "3           昨天  5年以上 本科及以上  https://m.liepin.com/job/1926567413.shtml   \n",
       "4           昨天  3年以上 本科及以上  https://m.liepin.com/job/1926420031.shtml   \n",
       "5           昨天   5年以上 统招本科  https://m.liepin.com/job/1926292629.shtml   \n",
       "6           昨天  3年以上 大专及以上  https://m.liepin.com/job/1925238817.shtml   \n",
       "7   2020-03-26   2年以上 学历不限      https://m.liepin.com/a/18569421.shtml   \n",
       "8           昨天   1年以上 统招本科  https://m.liepin.com/job/1924641017.shtml   \n",
       "9           昨天   1年以上 统招本科  https://m.liepin.com/job/1921434973.shtml   \n",
       "10          昨天   2年以上 统招本科      https://m.liepin.com/a/19077453.shtml   \n",
       "11          昨天   2年以上 统招本科      https://m.liepin.com/a/19077429.shtml   \n",
       "12          昨天   2年以上 统招本科      https://m.liepin.com/a/18784931.shtml   \n",
       "13          昨天  2年以上 本科及以上  https://m.liepin.com/job/1913960836.shtml   \n",
       "14          前天  1年以上 本科及以上  https://m.liepin.com/job/1927186335.shtml   \n",
       "15          前天  2年以上 本科及以上  https://m.liepin.com/job/1927025161.shtml   \n",
       "16          前天   5年以上 统招本科  https://m.liepin.com/job/1926885893.shtml   \n",
       "17          前天  3年以上 本科及以上  https://m.liepin.com/job/1926606033.shtml   \n",
       "18          前天  4年以上 大专及以上  https://m.liepin.com/job/1926501261.shtml   \n",
       "19          前天   经验不限 统招本科  https://m.liepin.com/job/1926109895.shtml   \n",
       "20          前天  1年以上 本科及以上  https://m.liepin.com/job/1925949165.shtml   \n",
       "21          前天  2年以上 本科及以上  https://m.liepin.com/job/1925922047.shtml   \n",
       "22          前天   2年以上 统招本科  https://m.liepin.com/job/1925414753.shtml   \n",
       "23          前天   3年以上 统招本科  https://m.liepin.com/job/1924656343.shtml   \n",
       "24          前天   5年以上 统招本科      https://m.liepin.com/a/19622953.shtml   \n",
       "25          前天   2年以上 统招本科      https://m.liepin.com/a/19618915.shtml   \n",
       "26          前天   3年以上 统招本科  https://m.liepin.com/job/1911819089.shtml   \n",
       "27  2020-04-01  2年以上 硕士及以上  https://m.liepin.com/job/1926968307.shtml   \n",
       "28  2020-04-01  2年以上 本科及以上  https://m.liepin.com/job/1925915631.shtml   \n",
       "29  2020-04-01   经验不限 统招本科  https://m.liepin.com/job/1922192733.shtml   \n",
       "30  2020-04-01   2年以上 统招本科  https://m.liepin.com/job/1919988491.shtml   \n",
       "31  2020-04-01  3年以上 本科及以上  https://m.liepin.com/job/1918854685.shtml   \n",
       "32  2020-04-01  2年以上 本科及以上  https://m.liepin.com/job/1917412855.shtml   \n",
       "33  2020-04-01   2年以上 统招本科  https://m.liepin.com/job/1917411127.shtml   \n",
       "34  2020-03-31  5年以上 本科及以上  https://m.liepin.com/job/1927112101.shtml   \n",
       "35  2020-03-31  3年以上 大专及以上  https://m.liepin.com/job/1926949151.shtml   \n",
       "36  2020-03-31  经验不限 本科及以上  https://m.liepin.com/job/1925880227.shtml   \n",
       "37  2020-03-31  2年以上 本科及以上  https://m.liepin.com/job/1922622715.shtml   \n",
       "38  2020-03-31   经验不限 学历不限  https://m.liepin.com/job/1920244061.shtml   \n",
       "39  2020-03-31  5年以上 本科及以上   https://m.liepin.com/job/199674161.shtml   \n",
       "40  2020-03-31  1年以上 本科及以上   https://m.liepin.com/job/198635044.shtml   \n",
       "41  2020-03-30  2年以上 本科及以上  https://m.liepin.com/job/1926948641.shtml   \n",
       "42  2020-03-30   2年以上 统招本科  https://m.liepin.com/job/1926195541.shtml   \n",
       "43  2020-03-30   3年以上 学历不限      https://m.liepin.com/a/19495749.shtml   \n",
       "44  2020-03-30   3年以上 统招本科      https://m.liepin.com/a/19347549.shtml   \n",
       "45  2020-03-30   1年以上 统招本科      https://m.liepin.com/a/19063297.shtml   \n",
       "46  2020-03-27   2年以上 统招本科  https://m.liepin.com/job/1926946115.shtml   \n",
       "47  2020-03-27  3年以上 本科及以上  https://m.liepin.com/job/1924672993.shtml   \n",
       "48  2020-03-26   2年以上 统招本科  https://m.liepin.com/job/1926903167.shtml   \n",
       "49  2020-03-26  3年以上 本科及以上  https://m.liepin.com/job/1923664395.shtml   \n",
       "50  2020-03-25  3年以上 本科及以上  https://m.liepin.com/job/1924692443.shtml   \n",
       "51  2020-03-25  1年以上 本科及以上  https://m.liepin.com/job/1920605757.shtml   \n",
       "52  2020-03-25   1年以上 学历不限      https://m.liepin.com/a/19376743.shtml   \n",
       "53  2020-03-23  经验不限 大专及以上  https://m.liepin.com/job/1922291239.shtml   \n",
       "54  2020-03-23  2年以上 本科及以上  https://m.liepin.com/job/1921893265.shtml   \n",
       "55  2020-03-23  经验不限 本科及以上  https://m.liepin.com/job/1917586595.shtml   \n",
       "56  2020-03-20   3年以上 统招本科  https://m.liepin.com/job/1926654193.shtml   \n",
       "57  2020-03-19  3年以上 本科及以上  https://m.liepin.com/job/1924683829.shtml   \n",
       "58  2020-03-19  3年以上 本科及以上  https://m.liepin.com/job/1923167241.shtml   \n",
       "59  2020-03-19  2年以上 大专及以上      https://m.liepin.com/a/19269411.shtml   \n",
       "\n",
       "                                     公司URL  \n",
       "0    https://m.liepin.com/company/8837861/  \n",
       "1   https://m.liepin.com/company/10094945/  \n",
       "2    https://m.liepin.com/company/9705250/  \n",
       "3     https://m.liepin.com/company/876848/  \n",
       "4    https://m.liepin.com/company/7870705/  \n",
       "5    https://m.liepin.com/company/8836020/  \n",
       "6    https://m.liepin.com/company/9660700/  \n",
       "7                                           \n",
       "8    https://m.liepin.com/company/8428035/  \n",
       "9    https://m.liepin.com/company/8082331/  \n",
       "10                                          \n",
       "11                                          \n",
       "12                                          \n",
       "13   https://m.liepin.com/company/8082331/  \n",
       "14    https://m.liepin.com/company/930104/  \n",
       "15  https://m.liepin.com/company/10255845/  \n",
       "16  https://m.liepin.com/company/10166945/  \n",
       "17   https://m.liepin.com/company/8037518/  \n",
       "18   https://m.liepin.com/company/9932713/  \n",
       "19   https://m.liepin.com/company/6780253/  \n",
       "20   https://m.liepin.com/company/9996995/  \n",
       "21   https://m.liepin.com/company/8880560/  \n",
       "22   https://m.liepin.com/company/8067064/  \n",
       "23  https://m.liepin.com/company/10178781/  \n",
       "24                                          \n",
       "25                                          \n",
       "26   https://m.liepin.com/company/9126093/  \n",
       "27   https://m.liepin.com/company/9903267/  \n",
       "28  https://m.liepin.com/company/10176153/  \n",
       "29  https://m.liepin.com/company/10141823/  \n",
       "30   https://m.liepin.com/company/1543625/  \n",
       "31   https://m.liepin.com/company/9656308/  \n",
       "32   https://m.liepin.com/company/9287281/  \n",
       "33   https://m.liepin.com/company/9287281/  \n",
       "34   https://m.liepin.com/company/3547412/  \n",
       "35  https://m.liepin.com/company/12179843/  \n",
       "36   https://m.liepin.com/company/8976402/  \n",
       "37   https://m.liepin.com/company/9638401/  \n",
       "38   https://m.liepin.com/company/9226181/  \n",
       "39   https://m.liepin.com/company/7892336/  \n",
       "40   https://m.liepin.com/company/7892336/  \n",
       "41   https://m.liepin.com/company/9770323/  \n",
       "42   https://m.liepin.com/company/9918971/  \n",
       "43                                          \n",
       "44                                          \n",
       "45                                          \n",
       "46   https://m.liepin.com/company/2485876/  \n",
       "47   https://m.liepin.com/company/8801986/  \n",
       "48  https://m.liepin.com/company/10094945/  \n",
       "49   https://m.liepin.com/company/8955475/  \n",
       "50   https://m.liepin.com/company/9284656/  \n",
       "51   https://m.liepin.com/company/9382271/  \n",
       "52                                          \n",
       "53  https://m.liepin.com/company/10056971/  \n",
       "54   https://m.liepin.com/company/9411137/  \n",
       "55   https://m.liepin.com/company/9287281/  \n",
       "56   https://m.liepin.com/company/2399664/  \n",
       "57   https://m.liepin.com/company/9284656/  \n",
       "58   https://m.liepin.com/company/9680336/  \n",
       "59                                          "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# C-1   单一页面\n",
    "url = \"https://m.liepin.com/zhaopin/?keyword=%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90%E5%B8%88\"\n",
    "from requests_html import HTMLSession\n",
    "session = HTMLSession()\n",
    "r = session.get( url )\n",
    "\n",
    "# C-5\n",
    "# 难: '公司URL', '时间', '经验'\n",
    "\n",
    "# 先取特定元素, 精准打击其子后辈\n",
    "主要元素 = r.html.xpath( \\\n",
    "    '//div[@class=\"job-card-wrap\"]//div[@class=\"job-card\"]')\n",
    "\n",
    "# 作为xpath字典，键为我要抓的牛肉名称，值为xpath\n",
    "dict_xpaths={ \n",
    "    'text': {\n",
    "        '经验':      './/ul/li[time]/text()'\n",
    "    },\n",
    "    'text_content': {\n",
    "        '职称':    './/ul/li/a[contains(@class,\"job-name\")]/span[@class=\"name-text\"]', \n",
    "        '薪水':    './/ul/li/a[contains(@class,\"job-name\")]/following-sibling::span', #获取弟节点的方法：following-sibling::元素名\n",
    "        '公司地点':'.//ul/li/time/following-sibling::a',                              #获取兄节点的方法：preceding-sibling::元素名\n",
    "        '公司名称': './/ul/li/a[contains(@class,\"company-name\")]', \n",
    "        '时间':    './/ul/li/time', \n",
    "    },\n",
    "    'href': {\n",
    "        '链结':    './/ul/li/a[contains(@class,\"job-name\")]', \n",
    "        '公司URL': './/ul/li/a[contains(@class,\"company-name\")]', #在class属性不止一个时，利用[contains(@class,\"指定class属性值\")]定位\n",
    "    }\n",
    "}\n",
    "\n",
    "def get_e_text_content(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]   \n",
    "    return(暂存结果)\n",
    "\n",
    "def get_e_text(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [\"\".join([x.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "    return(暂存结果)\n",
    "\n",
    "def get_e_href(_xpath_):\n",
    "    # 高级列表推导:如果URL存在（即len(xxx)>=1）则输出列表，否则输出一个空字符串\n",
    "    暂存结果 = [list(e.xpath(_xpath_, first=True).absolute_links)[0]\\\n",
    "               if len(e.xpath(_xpath_, first=True).absolute_links) >=1\\\n",
    "               else \"\" for e in 主要元素]\n",
    "    return(暂存结果)\n",
    "\n",
    "# 只对主要元素下进行.xpath取值\n",
    "数据字典 = dict()\n",
    "\n",
    "数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "\n",
    "数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "数据字典\n",
    "#print ([len(v) for k,v in 数据字典.items()])  # 檢查\n",
    "\n",
    "数据 = pd.DataFrame(数据字典)\n",
    "#数据.to_excel(\"20春_Web数据挖掘_week02_liepin.xlsx\", sheet_name=\"搜查结果\")\n",
    "数据 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 本周目标¶\n",
    "* 猎聘PC版\n",
    "* 上方导航有 公司行业 城市 薪资 的分页选单\n",
    "* 请练习xpath抽出数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 爬取猎聘PC版职位，薪水，公司等数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>时间</th>\n",
       "      <th>薪水</th>\n",
       "      <th>工作地点</th>\n",
       "      <th>学历</th>\n",
       "      <th>经验</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>职位</th>\n",
       "      <th>职位链接</th>\n",
       "      <th>公司链接</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>10-18k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>时代大数据</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1920605639.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9382271/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>数据中心-数据挖掘算法工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925789257.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>10-15k·12薪</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>2020校园招聘-数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1923453745.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>上海-长宁区</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>点评App部-机器学习/数据挖掘算法专家</td>\n",
       "      <td>https://www.liepin.com/job/1923289363.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>闪购-数据挖掘技术专家</td>\n",
       "      <td>https://www.liepin.com/job/1919432959.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>20-40k·13薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>高级数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1918060629.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>美团外卖-数据挖掘技术专家</td>\n",
       "      <td>https://www.liepin.com/job/1917876419.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>25-35k·14薪</td>\n",
       "      <td>kika公司</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>kika公司</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19593545.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>30-50k·16薪</td>\n",
       "      <td>深圳-南山区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>某综合性集团公司</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19577517.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>16-20k·15薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>国内知名金融保险公司</td>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>https://www.liepin.com/a/19468911.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>13-18k·12薪</td>\n",
       "      <td>长春-宽城区</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>某科技公司</td>\n",
       "      <td>大数据挖掘师</td>\n",
       "      <td>https://www.liepin.com/a/19443397.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>20-40k·14薪</td>\n",
       "      <td>杭州-西湖区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>某海外视频平台</td>\n",
       "      <td>高级数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19402695.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>45-65k·15薪</td>\n",
       "      <td>微博金融</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>微博金融</td>\n",
       "      <td>资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19243319.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>20-30k·12薪</td>\n",
       "      <td>成都</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>区块链和数据安全</td>\n",
       "      <td>网络安全大数据挖掘高级分析师</td>\n",
       "      <td>https://www.liepin.com/a/19208817.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>25-50k·16薪</td>\n",
       "      <td>深圳,上海,北京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>4年以上</td>\n",
       "      <td>地产百强</td>\n",
       "      <td>数据挖掘工程师/专家</td>\n",
       "      <td>https://www.liepin.com/a/18694767.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>30-60k·12薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>国内知名上市企业</td>\n",
       "      <td>资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/18525023.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>30-60k·13薪</td>\n",
       "      <td>北京-劲松</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>8年以上</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>数据挖掘专家</td>\n",
       "      <td>https://www.liepin.com/job/197599882.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>15-20k·13薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/194275684.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>35-50k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>北京三六九数动科技有限公司</td>\n",
       "      <td>风控数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927229905.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10204225/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>10-20k·12薪</td>\n",
       "      <td>柳州</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>1年以上</td>\n",
       "      <td>恩英汽车科技(上海)有限公司</td>\n",
       "      <td>数据挖掘与分析工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927212533.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9224435/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>15-20k·12薪</td>\n",
       "      <td>北京亿赛通科技发展有限责任公司</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>北京亿赛通科技发展有限责任公司</td>\n",
       "      <td>大数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927189899.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7885830/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>25-35k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>北京三六九数动科技有限公司</td>\n",
       "      <td>（高级）数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927160225.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10204225/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>网易集团</td>\n",
       "      <td>高级/资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927098131.shtml</td>\n",
       "      <td>https://www.liepin.com/company/5964833/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>15-30k·12薪</td>\n",
       "      <td>上海-徐汇区</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>上海猛犸信息服务有限公司</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927094945.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10095707/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>20-40k·16薪</td>\n",
       "      <td>杭州</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>网易集团</td>\n",
       "      <td>资深数据挖掘工程师-雷火用户体验研究中心001</td>\n",
       "      <td>https://www.liepin.com/job/1926571709.shtml</td>\n",
       "      <td>https://www.liepin.com/company/5964833/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>25-50k·12薪</td>\n",
       "      <td>深圳-科技园</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>高级数据挖掘工程师-腾讯音乐</td>\n",
       "      <td>https://www.liepin.com/job/1926393875.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>长沙</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844529.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>成都</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844527.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844525.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>重庆</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844521.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>南京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844519.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>天津</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844517.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>苏州</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844515.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>济南</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844513.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>杭州</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844511.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844509.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>武汉</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844507.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>上海</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844505.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844503.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>青岛</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844501.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             时间           薪水             工作地点     学历     经验             公司名称  \\\n",
       "0   2020年04月03日   10-18k·12薪               广州  本科及以上   2年以上            时代大数据   \n",
       "1   2020年04月05日           面议             美团点评  本科及以上  本科及以上             美团点评   \n",
       "2   2020年04月05日   10-15k·12薪             深演智能   统招本科   统招本科             深演智能   \n",
       "3   2020年04月05日           面议           上海-长宁区  本科及以上   3年以上             美团点评   \n",
       "4   2020年04月05日           面议             美团点评  本科及以上  本科及以上             美团点评   \n",
       "5   2020年04月05日   20-40k·13薪               北京  硕士及以上   3年以上             深演智能   \n",
       "6   2020年04月05日           面议             美团点评  本科及以上  本科及以上             美团点评   \n",
       "7   2020年04月05日   25-35k·14薪           kika公司   统招本科   统招本科           kika公司   \n",
       "8   2020年04月05日   30-50k·16薪           深圳-南山区   统招本科   3年以上         某综合性集团公司   \n",
       "9   2020年04月05日   16-20k·15薪               深圳   统招本科   2年以上       国内知名金融保险公司   \n",
       "10  2020年04月05日   13-18k·12薪           长春-宽城区   学历不限   2年以上            某科技公司   \n",
       "11  2020年04月05日   20-40k·14薪           杭州-西湖区   统招本科   3年以上          某海外视频平台   \n",
       "12  2020年04月05日   45-65k·15薪             微博金融   统招本科   统招本科             微博金融   \n",
       "13  2020年04月05日   20-30k·12薪               成都   统招本科   3年以上         区块链和数据安全   \n",
       "14  2020年04月05日   25-50k·16薪         深圳,上海,北京   统招本科   4年以上             地产百强   \n",
       "15  2020年04月05日   30-60k·12薪               深圳   统招本科   5年以上         国内知名上市企业   \n",
       "16  2020年04月05日   30-60k·13薪            北京-劲松  本科及以上   8年以上             深演智能   \n",
       "17  2020年04月05日   15-20k·13薪               北京  本科及以上   3年以上             深演智能   \n",
       "18  2020年04月04日   35-50k·12薪               北京  硕士及以上   3年以上    北京三六九数动科技有限公司   \n",
       "19  2020年04月04日   10-20k·12薪               柳州  本科及以上   1年以上   恩英汽车科技(上海)有限公司   \n",
       "20  2020年04月04日   15-20k·12薪  北京亿赛通科技发展有限责任公司  本科及以上  本科及以上  北京亿赛通科技发展有限责任公司   \n",
       "21  2020年04月04日   25-35k·12薪               北京  硕士及以上   2年以上    北京三六九数动科技有限公司   \n",
       "22  2020年04月04日           面议               杭州  硕士及以上   3年以上             网易集团   \n",
       "23  2020年04月04日   15-30k·12薪           上海-徐汇区  本科及以上   2年以上     上海猛犸信息服务有限公司   \n",
       "24  2020年04月04日   20-40k·16薪               杭州  硕士及以上   3年以上             网易集团   \n",
       "25  2020年04月04日   25-50k·12薪           深圳-科技园  硕士及以上   3年以上               腾讯   \n",
       "26  2020年04月04日  70-100k·12薪               长沙   统招本科   5年以上             卓尔人人   \n",
       "27  2020年04月04日  70-100k·12薪               成都   统招本科   5年以上             卓尔人人   \n",
       "28  2020年04月04日  70-100k·12薪               广州   统招本科   5年以上             卓尔人人   \n",
       "29  2020年04月04日  70-100k·12薪               重庆   统招本科   5年以上             卓尔人人   \n",
       "30  2020年04月04日  70-100k·12薪               南京   统招本科   5年以上             卓尔人人   \n",
       "31  2020年04月04日  70-100k·12薪               天津   统招本科   5年以上             卓尔人人   \n",
       "32  2020年04月04日  70-100k·12薪               苏州   统招本科   5年以上             卓尔人人   \n",
       "33  2020年04月04日  70-100k·12薪               济南   统招本科   5年以上             卓尔人人   \n",
       "34  2020年04月04日  70-100k·12薪               杭州   统招本科   5年以上             卓尔人人   \n",
       "35  2020年04月04日  70-100k·12薪               深圳   统招本科   5年以上             卓尔人人   \n",
       "36  2020年04月04日  70-100k·12薪               武汉   统招本科   5年以上             卓尔人人   \n",
       "37  2020年04月04日  70-100k·12薪               上海   统招本科   5年以上             卓尔人人   \n",
       "38  2020年04月04日  70-100k·12薪               北京   统招本科   5年以上             卓尔人人   \n",
       "39  2020年04月04日  70-100k·12薪               青岛   统招本科   5年以上             卓尔人人   \n",
       "\n",
       "                         职位                                         职位链接  \\\n",
       "0                   数据挖掘工程师  https://www.liepin.com/job/1920605639.shtml   \n",
       "1            数据中心-数据挖掘算法工程师  https://www.liepin.com/job/1925789257.shtml   \n",
       "2          2020校园招聘-数据挖掘工程师  https://www.liepin.com/job/1923453745.shtml   \n",
       "3      点评App部-机器学习/数据挖掘算法专家  https://www.liepin.com/job/1923289363.shtml   \n",
       "4               闪购-数据挖掘技术专家  https://www.liepin.com/job/1919432959.shtml   \n",
       "5                 高级数据挖掘工程师  https://www.liepin.com/job/1918060629.shtml   \n",
       "6             美团外卖-数据挖掘技术专家  https://www.liepin.com/job/1917876419.shtml   \n",
       "7                   数据挖掘工程师      https://www.liepin.com/a/19593545.shtml   \n",
       "8                   数据挖掘工程师      https://www.liepin.com/a/19577517.shtml   \n",
       "9                      数据挖掘      https://www.liepin.com/a/19468911.shtml   \n",
       "10                   大数据挖掘师      https://www.liepin.com/a/19443397.shtml   \n",
       "11                高级数据挖掘工程师      https://www.liepin.com/a/19402695.shtml   \n",
       "12                资深数据挖掘工程师      https://www.liepin.com/a/19243319.shtml   \n",
       "13           网络安全大数据挖掘高级分析师      https://www.liepin.com/a/19208817.shtml   \n",
       "14               数据挖掘工程师/专家      https://www.liepin.com/a/18694767.shtml   \n",
       "15                资深数据挖掘工程师      https://www.liepin.com/a/18525023.shtml   \n",
       "16                   数据挖掘专家   https://www.liepin.com/job/197599882.shtml   \n",
       "17                资深数据挖掘工程师   https://www.liepin.com/job/194275684.shtml   \n",
       "18                风控数据挖掘工程师  https://www.liepin.com/job/1927229905.shtml   \n",
       "19               数据挖掘与分析工程师  https://www.liepin.com/job/1927212533.shtml   \n",
       "20                 大数据挖掘工程师  https://www.liepin.com/job/1927189899.shtml   \n",
       "21              （高级）数据挖掘工程师  https://www.liepin.com/job/1927160225.shtml   \n",
       "22             高级/资深数据挖掘工程师  https://www.liepin.com/job/1927098131.shtml   \n",
       "23                  数据挖掘工程师  https://www.liepin.com/job/1927094945.shtml   \n",
       "24  资深数据挖掘工程师-雷火用户体验研究中心001  https://www.liepin.com/job/1926571709.shtml   \n",
       "25           高级数据挖掘工程师-腾讯音乐  https://www.liepin.com/job/1926393875.shtml   \n",
       "26                  数据挖掘工程师  https://www.liepin.com/job/1925844529.shtml   \n",
       "27                  数据挖掘工程师  https://www.liepin.com/job/1925844527.shtml   \n",
       "28                  数据挖掘工程师  https://www.liepin.com/job/1925844525.shtml   \n",
       "29                  数据挖掘工程师  https://www.liepin.com/job/1925844521.shtml   \n",
       "30                  数据挖掘工程师  https://www.liepin.com/job/1925844519.shtml   \n",
       "31                  数据挖掘工程师  https://www.liepin.com/job/1925844517.shtml   \n",
       "32                  数据挖掘工程师  https://www.liepin.com/job/1925844515.shtml   \n",
       "33                  数据挖掘工程师  https://www.liepin.com/job/1925844513.shtml   \n",
       "34                  数据挖掘工程师  https://www.liepin.com/job/1925844511.shtml   \n",
       "35                  数据挖掘工程师  https://www.liepin.com/job/1925844509.shtml   \n",
       "36                  数据挖掘工程师  https://www.liepin.com/job/1925844507.shtml   \n",
       "37                  数据挖掘工程师  https://www.liepin.com/job/1925844505.shtml   \n",
       "38                  数据挖掘工程师  https://www.liepin.com/job/1925844503.shtml   \n",
       "39                  数据挖掘工程师  https://www.liepin.com/job/1925844501.shtml   \n",
       "\n",
       "                                        公司链接  \n",
       "0    https://www.liepin.com/company/9382271/  \n",
       "1    https://www.liepin.com/company/2036768/  \n",
       "2    https://www.liepin.com/company/1157027/  \n",
       "3    https://www.liepin.com/company/2036768/  \n",
       "4    https://www.liepin.com/company/2036768/  \n",
       "5    https://www.liepin.com/company/1157027/  \n",
       "6    https://www.liepin.com/company/2036768/  \n",
       "7                                             \n",
       "8                                             \n",
       "9                                             \n",
       "10                                            \n",
       "11                                            \n",
       "12                                            \n",
       "13                                            \n",
       "14                                            \n",
       "15                                            \n",
       "16   https://www.liepin.com/company/1157027/  \n",
       "17   https://www.liepin.com/company/1157027/  \n",
       "18  https://www.liepin.com/company/10204225/  \n",
       "19   https://www.liepin.com/company/9224435/  \n",
       "20   https://www.liepin.com/company/7885830/  \n",
       "21  https://www.liepin.com/company/10204225/  \n",
       "22   https://www.liepin.com/company/5964833/  \n",
       "23  https://www.liepin.com/company/10095707/  \n",
       "24   https://www.liepin.com/company/5964833/  \n",
       "25   https://www.liepin.com/company/7983148/  \n",
       "26  https://www.liepin.com/company/12146335/  \n",
       "27  https://www.liepin.com/company/12146335/  \n",
       "28  https://www.liepin.com/company/12146335/  \n",
       "29  https://www.liepin.com/company/12146335/  \n",
       "30  https://www.liepin.com/company/12146335/  \n",
       "31  https://www.liepin.com/company/12146335/  \n",
       "32  https://www.liepin.com/company/12146335/  \n",
       "33  https://www.liepin.com/company/12146335/  \n",
       "34  https://www.liepin.com/company/12146335/  \n",
       "35  https://www.liepin.com/company/12146335/  \n",
       "36  https://www.liepin.com/company/12146335/  \n",
       "37  https://www.liepin.com/company/12146335/  \n",
       "38  https://www.liepin.com/company/12146335/  \n",
       "39  https://www.liepin.com/company/12146335/  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from requests_html import HTMLSession\n",
    "import pandas as pd\n",
    "import lxml\n",
    "session= HTMLSession()\n",
    "\n",
    "url=\"https://www.liepin.com/zhaopin/?industries=&subIndustry=&dqs=&salary=&jobKind=&pubTime=&compkind=&compscale=&industryType=&searchType=1&clean_condition=&isAnalysis=&init=1&sortFlag=15&flushckid=0&fromSearchBtn=1&headckid=52b1c0be834fc722&d_headId=41c0cceeae9b7e1f5aec9cc84ba87032&d_ckId=41c0cceeae9b7e1f5aec9cc84ba87032&d_sfrom=search_prime&d_curPage=0&d_pageSize=40&siTag=bFGQTbwE_AAQSb-u11jrBw%7EfA9rXquZc5IkJpXC-Ycixw&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98\"\n",
    "r= session.get(url)\n",
    "#设置主要的xpath路径\n",
    "main= r.html.xpath(\"//div[contains(@class,'sojob-item-main')]\")\n",
    "#使用字典存取特定的数据的位置\n",
    "dict_xpath={\n",
    "    \"text\":{\n",
    "        \"职位\":\".//div[@class='job-info']/h3/a/text()\"\n",
    "    },\n",
    "    \"text_content\":{\n",
    "        \"时间\":\".//div[@class='job-info']/p[contains(@class,'time-info')]/time/@title\",\n",
    "        \n",
    "        \"薪水\":\".//div[@class='job-info']/p/span[@class='text-warning']/text()\",\n",
    "        \"工作地点\":\".//p[contains(@class,condition)]/a/text()\",  #/div[@class='job-info']      [@class='area']\n",
    "        \"学历\":\".//span[@class='edu']/text()\",\n",
    "        \"经验\":\".//p[contains(@class,condition)]/span[3]/text()\",\n",
    "        \"公司名称\":\".//p[contains(@class,'company-name')]/a/text()\"\n",
    "    },\n",
    "    \"href\":{\n",
    "        \"职位链接\":\".//div[@class='job-info']/h3/a\",\n",
    "        \"公司链接\":\"//div[contains(@class,'sojob-item-main')]//p[@class='company-name']/a\" #/div[contains(@class,'company-info')]\n",
    "    }\n",
    "    \n",
    "}\n",
    "\n",
    "def get_e_text(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [\"\".join([x.strip() for x in e.xpath(_xpath_)]) for e in main]\n",
    "    return(暂存结果)\n",
    "\n",
    "def get_text_content(_xpath_):\n",
    "    # 高级列表推导\n",
    "    \n",
    "    暂存结果 =[ e.xpath(_xpath_)[0]  for e in main]\n",
    "    \n",
    "    #暂存结果 =main#.xpath(_xpath_)\n",
    "    return(暂存结果)          \n",
    "\n",
    "#如果为真，first=True返回第一个元素\n",
    "\n",
    "def get_href(_xpath_):          \n",
    "    暂存结果=[list(e.xpath(_xpath_)[0].absolute_links)[0]              \n",
    "                  if len(list(e.xpath(_xpath_)[0].absolute_links))>=1\n",
    "                        else \"\"  for e in main  ]\n",
    "          \n",
    "\n",
    "    return(暂存结果)\n",
    "数据字典=dict()\n",
    "数据字典={k:get_text_content(v) for k,v in dict_xpath[\"text_content\"].items()}\n",
    "数据字典.update({k:get_e_text(v) for k,v in dict_xpath['text'].items()})\n",
    "数据字典.update({k:get_href(v) for k,v in dict_xpath['href'].items() })\n",
    "数据字典\n",
    "df=pd.DataFrame(数据字典)\n",
    "df\n",
    "#with open('猎聘网数据挖掘方面招聘信息.xlsx' ,encoding = \"utf8\", mode = 'a') as fp:\n",
    "#    df.to_excel(\"猎聘网数据挖掘方面招聘信息.xlsx\", sheet_name=\"猎聘PC\")\n",
    "#数据\n",
    "#eweqw ={k:get_href(v) for k,v in dict_xpath[\"href\"].items() }\n",
    "#eweqw\n",
    "#for k,v in dict_xpath[\"text\"].items():\n",
    "#       print(k,get_e_text(v) )\n",
    "#for k,v in dict_xpath[\"href\"].items():\n",
    "#       print(k,get_href(v) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用urllib3 解析 url¶\n",
    "* 上面的url应该触动不同的页面查询，但能不能轻松无误的拆分url并进行比较？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### urllib模块功能介绍¶\n",
    "* urlparse 返回的6个部分，分别是：scheme(机制)丶netloc(网络位置)丶path(路径)丶params(路径段参数)丶query(查询)丶fragment(片段)。\n",
    "parse_qs 返回query(查询)多个部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#使用 urllib.parse 解析\n",
    "from urllib.parse import urlparse, parse_qs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 取出特定URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'中国500强': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=155&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f',\n",
       " '2018互联网300强': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=182&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f',\n",
       " '制造业500强': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=186&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f',\n",
       " 'AI创新成长50强 ': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=189&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f',\n",
       " '独角兽': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=130&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f',\n",
       " '上市公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=156&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f'}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company=r.html.xpath('//div[@data-selector=\"search-conditions\"]')[0] \\\n",
    "                    .xpath('//dt[@class=\"search-title\"]/following-sibling::dd')[0] \\\n",
    "                    .xpath('//div[contains(@class,\"hot-comp-tags\")]/a')\n",
    "company_lianjie={x.xpath('a/text()')[0]:x.xpath('a/@href')[0]  for x in company}  #输出的是一个列表，[0]取出值\n",
    "company_lianjie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分解URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ParseResult(scheme='', netloc='', path='/zhaopin/', params='', query='isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=155&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f', fragment=''),\n",
       " ParseResult(scheme='', netloc='', path='/zhaopin/', params='', query='isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=182&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f', fragment=''),\n",
       " ParseResult(scheme='', netloc='', path='/zhaopin/', params='', query='isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=186&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f', fragment=''),\n",
       " ParseResult(scheme='', netloc='', path='/zhaopin/', params='', query='isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=189&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f', fragment=''),\n",
       " ParseResult(scheme='', netloc='', path='/zhaopin/', params='', query='isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=130&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f', fragment=''),\n",
       " ParseResult(scheme='', netloc='', path='/zhaopin/', params='', query='isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&compkind=&fromSearchBtn=2&compTag=156&sortFlag=15&ckid=970029fdeea53da8&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f', fragment='')]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#取出键company_lianjie的值，利用for循环对所有url进行拆解  \n",
    "[ urlparse(x) for x in company_lianjie.values()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### nunique()相异值计量比对\n",
    "* 使用 pd.DataFrame进行 nunique()相异值计量比对  \n",
    "* Pandas nunique() 用于获取唯一值的统计次数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scheme</th>\n",
       "      <th>netloc</th>\n",
       "      <th>path</th>\n",
       "      <th>params</th>\n",
       "      <th>query</th>\n",
       "      <th>fragment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>isAnalysis=&amp;dqs=&amp;pubTime=&amp;salary=&amp;subIndustry=...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>isAnalysis=&amp;dqs=&amp;pubTime=&amp;salary=&amp;subIndustry=...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>isAnalysis=&amp;dqs=&amp;pubTime=&amp;salary=&amp;subIndustry=...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>isAnalysis=&amp;dqs=&amp;pubTime=&amp;salary=&amp;subIndustry=...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>isAnalysis=&amp;dqs=&amp;pubTime=&amp;salary=&amp;subIndustry=...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>isAnalysis=&amp;dqs=&amp;pubTime=&amp;salary=&amp;subIndustry=...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  scheme netloc       path params  \\\n",
       "0                /zhaopin/          \n",
       "1                /zhaopin/          \n",
       "2                /zhaopin/          \n",
       "3                /zhaopin/          \n",
       "4                /zhaopin/          \n",
       "5                /zhaopin/          \n",
       "\n",
       "                                               query fragment  \n",
       "0  isAnalysis=&dqs=&pubTime=&salary=&subIndustry=...           \n",
       "1  isAnalysis=&dqs=&pubTime=&salary=&subIndustry=...           \n",
       "2  isAnalysis=&dqs=&pubTime=&salary=&subIndustry=...           \n",
       "3  isAnalysis=&dqs=&pubTime=&salary=&subIndustry=...           \n",
       "4  isAnalysis=&dqs=&pubTime=&salary=&subIndustry=...           \n",
       "5  isAnalysis=&dqs=&pubTime=&salary=&subIndustry=...           "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.DataFrame([ urlparse(x) for x in company_lianjie.values()])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6 entries, 0 to 5\n",
      "Data columns (total 6 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   scheme    6 non-null      object\n",
      " 1   netloc    6 non-null      object\n",
      " 2   path      6 non-null      object\n",
      " 3   params    6 non-null      object\n",
      " 4   query     6 non-null      object\n",
      " 5   fragment  6 non-null      object\n",
      "dtypes: object(6)\n",
      "memory usage: 416.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scheme      1\n",
       "netloc      1\n",
       "path        1\n",
       "params      1\n",
       "query       6\n",
       "fragment    1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  针对query 再解析之 \n",
    "* urllib.parse.parse_qs 返回字典 ,其值为列表\n",
    "* urllib.parse.parse_qsl 返回列表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key              1\n",
      "init             1\n",
      "searchType       1\n",
      "headckid         1\n",
      "flushckid        1\n",
      "fromSearchBtn    1\n",
      "compTag          6\n",
      "sortFlag         1\n",
      "ckid             1\n",
      "siTag            1\n",
      "d_sfrom          1\n",
      "d_ckId           1\n",
      "d_curPage        1\n",
      "d_pageSize       1\n",
      "d_headId         1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>compTag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    key compTag\n",
       "0  数据挖掘     155\n",
       "1  数据挖掘     182\n",
       "2  数据挖掘     186\n",
       "3  数据挖掘     189\n",
       "4  数据挖掘     130\n",
       "5  数据挖掘     156"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#利用parse_qs进行解析\n",
    "#for循环取出query的字符串，再用字典推导式，parse_qs把url解析成列表，键为索引，值为url   \n",
    "df_qs=pd.DataFrame([{k:v[0] for k,v in parse_qs(e).items()}for e in df['query']])\n",
    "df_qs\n",
    "print(df_qs.nunique())\n",
    "df_qs[['key','compTag']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  小结\n",
    "* comTag 是不同的公司选择器, 数值不样, 对映到不同类型的公司\n",
    "* keyword 是搜查关键字"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建构 参数模板 及 字典_compTag\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key': ['数据挖掘'], 'init': ['-1'], 'searchType': ['1'], 'headckid': ['970029fdeea53da8'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'compTag': ['155'], 'sortFlag': ['15'], 'ckid': ['970029fdeea53da8'], 'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_prime'], 'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f']}\n"
     ]
    }
   ],
   "source": [
    "def parse_url_qs_for_compTag (url):\n",
    "    six_parts = urlparse(url)    #利用urlparse分解url\n",
    "    out = parse_qs(six_parts.query)#利用parse_qs拆解query的属性值\n",
    "    return (out)\n",
    "# parse_url_qs_for_compTag(list(公司数据选择器链结.values())[0])['compTag']\n",
    "参数模板 = parse_url_qs_for_compTag(list(company_lianjie.values())[0])#取url的字符串\n",
    "print(参数模板)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'中国500强': '155', '2018互联网300强': '182', '制造业500强': '186', 'AI创新成长50强 ': '189', '独角兽': '130', '上市公司': '156'}\n"
     ]
    }
   ],
   "source": [
    "字典_compTag = { k:parse_url_qs_for_compTag(v)['compTag'][0] for k,v in company_lianjie.items()}#把url进行拆解，然后取出键为compTag的值，形成新的字典\n",
    "print (字典_compTag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'2018互联网300强': {'ckid': ['970029fdeea53da8'],\n",
      "                 'compTag': ['182'],\n",
      "                 'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                 'd_curPage': ['0'],\n",
      "                 'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                 'd_pageSize': ['40'],\n",
      "                 'd_sfrom': ['search_prime'],\n",
      "                 'flushckid': ['1'],\n",
      "                 'fromSearchBtn': ['2'],\n",
      "                 'headckid': ['970029fdeea53da8'],\n",
      "                 'init': ['-1'],\n",
      "                 'key': ['数据挖掘'],\n",
      "                 'keyword': ['用户体验'],\n",
      "                 'searchType': ['1'],\n",
      "                 'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                 'sortFlag': ['15']},\n",
      " 'AI创新成长50强 ': {'ckid': ['970029fdeea53da8'],\n",
      "                'compTag': ['189'],\n",
      "                'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                'd_curPage': ['0'],\n",
      "                'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                'd_pageSize': ['40'],\n",
      "                'd_sfrom': ['search_prime'],\n",
      "                'flushckid': ['1'],\n",
      "                'fromSearchBtn': ['2'],\n",
      "                'headckid': ['970029fdeea53da8'],\n",
      "                'init': ['-1'],\n",
      "                'key': ['数据挖掘'],\n",
      "                'keyword': ['用户体验'],\n",
      "                'searchType': ['1'],\n",
      "                'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                'sortFlag': ['15']},\n",
      " '上市公司': {'ckid': ['970029fdeea53da8'],\n",
      "          'compTag': ['156'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['数据挖掘'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '中国500强': {'ckid': ['970029fdeea53da8'],\n",
      "            'compTag': ['155'],\n",
      "            'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "            'd_curPage': ['0'],\n",
      "            'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "            'd_pageSize': ['40'],\n",
      "            'd_sfrom': ['search_prime'],\n",
      "            'flushckid': ['1'],\n",
      "            'fromSearchBtn': ['2'],\n",
      "            'headckid': ['970029fdeea53da8'],\n",
      "            'init': ['-1'],\n",
      "            'key': ['数据挖掘'],\n",
      "            'keyword': ['用户体验'],\n",
      "            'searchType': ['1'],\n",
      "            'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "            'sortFlag': ['15']},\n",
      " '制造业500强': {'ckid': ['970029fdeea53da8'],\n",
      "             'compTag': ['186'],\n",
      "             'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "             'd_curPage': ['0'],\n",
      "             'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "             'd_pageSize': ['40'],\n",
      "             'd_sfrom': ['search_prime'],\n",
      "             'flushckid': ['1'],\n",
      "             'fromSearchBtn': ['2'],\n",
      "             'headckid': ['970029fdeea53da8'],\n",
      "             'init': ['-1'],\n",
      "             'key': ['数据挖掘'],\n",
      "             'keyword': ['用户体验'],\n",
      "             'searchType': ['1'],\n",
      "             'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "             'sortFlag': ['15']},\n",
      " '独角兽': {'ckid': ['970029fdeea53da8'],\n",
      "         'compTag': ['130'],\n",
      "         'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_curPage': ['0'],\n",
      "         'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_pageSize': ['40'],\n",
      "         'd_sfrom': ['search_prime'],\n",
      "         'flushckid': ['1'],\n",
      "         'fromSearchBtn': ['2'],\n",
      "         'headckid': ['970029fdeea53da8'],\n",
      "         'init': ['-1'],\n",
      "         'key': ['数据挖掘'],\n",
      "         'keyword': ['用户体验'],\n",
      "         'searchType': ['1'],\n",
      "         'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "         'sortFlag': ['15']}}\n"
     ]
    }
   ],
   "source": [
    "# 建构 参数模板  \n",
    "import pprint\n",
    "def 参数模板生成(compTag , keyword ):\n",
    "    参数 = 参数模板.copy()    #对url的query进行拆解\n",
    "    参数['compTag'] = compTag#形式参数\n",
    "    参数['keyword'] = keyword#形式参数\n",
    "    return (参数)\n",
    "#compTag = [v], keyword = ['用户体验']   compTag，keyword是形参\n",
    "参数_compTag_用户体验 = { k:参数模板生成(compTag = [v], keyword = ['用户体验']) for k,v in 字典_compTag.items()}\n",
    "pprint.pprint(参数_compTag_用户体验)               "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## requests生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key': ['数据挖掘'], 'init': ['-1'], 'searchType': ['1'], 'headckid': ['970029fdeea53da8'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'compTag': ['156'], 'sortFlag': ['15'], 'ckid': ['970029fdeea53da8'], 'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_prime'], 'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'], 'keyword': ['用户体验']}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://www.liepin.com/zhaopin/?key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&fromSearchBtn=2&compTag=156&sortFlag=15&ckid=970029fdeea53da8&siTag=LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f&keyword=%E7%94%A8%E6%88%B7%E4%BD%93%E9%AA%8C'"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#多个页面准备测试1 世界500强\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "session = HTMLSession()\n",
    "payload = 参数_compTag_用户体验['上市公司']\n",
    "print(payload)\n",
    "r = session.get( url, params = payload)\n",
    "r.url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用函数进行数据爬取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 单个页面的爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>时间</th>\n",
       "      <th>薪水</th>\n",
       "      <th>工作地点</th>\n",
       "      <th>学历</th>\n",
       "      <th>经验</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>职位</th>\n",
       "      <th>职位链接</th>\n",
       "      <th>公司链接</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>10-18k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>时代大数据</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1920605639.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9382271/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>数据中心-数据挖掘算法工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925789257.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>10-15k·12薪</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>2020校园招聘-数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1923453745.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>上海-长宁区</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>点评App部-机器学习/数据挖掘算法专家</td>\n",
       "      <td>https://www.liepin.com/job/1923289363.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>闪购-数据挖掘技术专家</td>\n",
       "      <td>https://www.liepin.com/job/1919432959.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>20-40k·13薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>高级数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1918060629.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>美团外卖-数据挖掘技术专家</td>\n",
       "      <td>https://www.liepin.com/job/1917876419.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>25-35k·14薪</td>\n",
       "      <td>kika公司</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>kika公司</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19593545.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>30-50k·16薪</td>\n",
       "      <td>深圳-南山区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>某综合性集团公司</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19577517.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>16-20k·15薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>国内知名金融保险公司</td>\n",
       "      <td>数据挖掘</td>\n",
       "      <td>https://www.liepin.com/a/19468911.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>13-18k·12薪</td>\n",
       "      <td>长春-宽城区</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>某科技公司</td>\n",
       "      <td>大数据挖掘师</td>\n",
       "      <td>https://www.liepin.com/a/19443397.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>20-40k·14薪</td>\n",
       "      <td>杭州-西湖区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>某海外视频平台</td>\n",
       "      <td>高级数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19402695.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>45-65k·15薪</td>\n",
       "      <td>微博金融</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>微博金融</td>\n",
       "      <td>资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/19243319.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>20-30k·12薪</td>\n",
       "      <td>成都</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>区块链和数据安全</td>\n",
       "      <td>网络安全大数据挖掘高级分析师</td>\n",
       "      <td>https://www.liepin.com/a/19208817.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>25-50k·16薪</td>\n",
       "      <td>深圳,上海,北京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>4年以上</td>\n",
       "      <td>地产百强</td>\n",
       "      <td>数据挖掘工程师/专家</td>\n",
       "      <td>https://www.liepin.com/a/18694767.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>30-60k·12薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>国内知名上市企业</td>\n",
       "      <td>资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/a/18525023.shtml</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>30-60k·13薪</td>\n",
       "      <td>北京-劲松</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>8年以上</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>数据挖掘专家</td>\n",
       "      <td>https://www.liepin.com/job/197599882.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>15-20k·13薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/194275684.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>35-50k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>北京三六九数动科技有限公司</td>\n",
       "      <td>风控数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927229905.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10204225/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>10-20k·12薪</td>\n",
       "      <td>柳州</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>1年以上</td>\n",
       "      <td>恩英汽车科技(上海)有限公司</td>\n",
       "      <td>数据挖掘与分析工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927212533.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9224435/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>15-20k·12薪</td>\n",
       "      <td>北京亿赛通科技发展有限责任公司</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>北京亿赛通科技发展有限责任公司</td>\n",
       "      <td>大数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927189899.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7885830/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>25-35k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>北京三六九数动科技有限公司</td>\n",
       "      <td>（高级）数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927160225.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10204225/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>网易集团</td>\n",
       "      <td>高级/资深数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927098131.shtml</td>\n",
       "      <td>https://www.liepin.com/company/5964833/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>15-30k·12薪</td>\n",
       "      <td>上海-徐汇区</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>上海猛犸信息服务有限公司</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1927094945.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10095707/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>20-40k·16薪</td>\n",
       "      <td>杭州</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>网易集团</td>\n",
       "      <td>资深数据挖掘工程师-雷火用户体验研究中心001</td>\n",
       "      <td>https://www.liepin.com/job/1926571709.shtml</td>\n",
       "      <td>https://www.liepin.com/company/5964833/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>25-50k·12薪</td>\n",
       "      <td>深圳-科技园</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>高级数据挖掘工程师-腾讯音乐</td>\n",
       "      <td>https://www.liepin.com/job/1926393875.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>长沙</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844529.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>成都</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844527.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844525.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>重庆</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844521.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>南京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844519.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>天津</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844517.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>苏州</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844515.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>济南</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844513.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>杭州</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844511.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844509.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>武汉</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844507.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>上海</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844505.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844503.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>青岛</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844501.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             时间           薪水             工作地点     学历     经验             公司名称  \\\n",
       "0   2020年04月03日   10-18k·12薪               广州  本科及以上   2年以上            时代大数据   \n",
       "1   2020年04月05日           面议             美团点评  本科及以上  本科及以上             美团点评   \n",
       "2   2020年04月05日   10-15k·12薪             深演智能   统招本科   统招本科             深演智能   \n",
       "3   2020年04月05日           面议           上海-长宁区  本科及以上   3年以上             美团点评   \n",
       "4   2020年04月05日           面议             美团点评  本科及以上  本科及以上             美团点评   \n",
       "5   2020年04月05日   20-40k·13薪               北京  硕士及以上   3年以上             深演智能   \n",
       "6   2020年04月05日           面议             美团点评  本科及以上  本科及以上             美团点评   \n",
       "7   2020年04月05日   25-35k·14薪           kika公司   统招本科   统招本科           kika公司   \n",
       "8   2020年04月05日   30-50k·16薪           深圳-南山区   统招本科   3年以上         某综合性集团公司   \n",
       "9   2020年04月05日   16-20k·15薪               深圳   统招本科   2年以上       国内知名金融保险公司   \n",
       "10  2020年04月05日   13-18k·12薪           长春-宽城区   学历不限   2年以上            某科技公司   \n",
       "11  2020年04月05日   20-40k·14薪           杭州-西湖区   统招本科   3年以上          某海外视频平台   \n",
       "12  2020年04月05日   45-65k·15薪             微博金融   统招本科   统招本科             微博金融   \n",
       "13  2020年04月05日   20-30k·12薪               成都   统招本科   3年以上         区块链和数据安全   \n",
       "14  2020年04月05日   25-50k·16薪         深圳,上海,北京   统招本科   4年以上             地产百强   \n",
       "15  2020年04月05日   30-60k·12薪               深圳   统招本科   5年以上         国内知名上市企业   \n",
       "16  2020年04月05日   30-60k·13薪            北京-劲松  本科及以上   8年以上             深演智能   \n",
       "17  2020年04月05日   15-20k·13薪               北京  本科及以上   3年以上             深演智能   \n",
       "18  2020年04月04日   35-50k·12薪               北京  硕士及以上   3年以上    北京三六九数动科技有限公司   \n",
       "19  2020年04月04日   10-20k·12薪               柳州  本科及以上   1年以上   恩英汽车科技(上海)有限公司   \n",
       "20  2020年04月04日   15-20k·12薪  北京亿赛通科技发展有限责任公司  本科及以上  本科及以上  北京亿赛通科技发展有限责任公司   \n",
       "21  2020年04月04日   25-35k·12薪               北京  硕士及以上   2年以上    北京三六九数动科技有限公司   \n",
       "22  2020年04月04日           面议               杭州  硕士及以上   3年以上             网易集团   \n",
       "23  2020年04月04日   15-30k·12薪           上海-徐汇区  本科及以上   2年以上     上海猛犸信息服务有限公司   \n",
       "24  2020年04月04日   20-40k·16薪               杭州  硕士及以上   3年以上             网易集团   \n",
       "25  2020年04月04日   25-50k·12薪           深圳-科技园  硕士及以上   3年以上               腾讯   \n",
       "26  2020年04月04日  70-100k·12薪               长沙   统招本科   5年以上             卓尔人人   \n",
       "27  2020年04月04日  70-100k·12薪               成都   统招本科   5年以上             卓尔人人   \n",
       "28  2020年04月04日  70-100k·12薪               广州   统招本科   5年以上             卓尔人人   \n",
       "29  2020年04月04日  70-100k·12薪               重庆   统招本科   5年以上             卓尔人人   \n",
       "30  2020年04月04日  70-100k·12薪               南京   统招本科   5年以上             卓尔人人   \n",
       "31  2020年04月04日  70-100k·12薪               天津   统招本科   5年以上             卓尔人人   \n",
       "32  2020年04月04日  70-100k·12薪               苏州   统招本科   5年以上             卓尔人人   \n",
       "33  2020年04月04日  70-100k·12薪               济南   统招本科   5年以上             卓尔人人   \n",
       "34  2020年04月04日  70-100k·12薪               杭州   统招本科   5年以上             卓尔人人   \n",
       "35  2020年04月04日  70-100k·12薪               深圳   统招本科   5年以上             卓尔人人   \n",
       "36  2020年04月04日  70-100k·12薪               武汉   统招本科   5年以上             卓尔人人   \n",
       "37  2020年04月04日  70-100k·12薪               上海   统招本科   5年以上             卓尔人人   \n",
       "38  2020年04月04日  70-100k·12薪               北京   统招本科   5年以上             卓尔人人   \n",
       "39  2020年04月04日  70-100k·12薪               青岛   统招本科   5年以上             卓尔人人   \n",
       "\n",
       "                         职位                                         职位链接  \\\n",
       "0                   数据挖掘工程师  https://www.liepin.com/job/1920605639.shtml   \n",
       "1            数据中心-数据挖掘算法工程师  https://www.liepin.com/job/1925789257.shtml   \n",
       "2          2020校园招聘-数据挖掘工程师  https://www.liepin.com/job/1923453745.shtml   \n",
       "3      点评App部-机器学习/数据挖掘算法专家  https://www.liepin.com/job/1923289363.shtml   \n",
       "4               闪购-数据挖掘技术专家  https://www.liepin.com/job/1919432959.shtml   \n",
       "5                 高级数据挖掘工程师  https://www.liepin.com/job/1918060629.shtml   \n",
       "6             美团外卖-数据挖掘技术专家  https://www.liepin.com/job/1917876419.shtml   \n",
       "7                   数据挖掘工程师      https://www.liepin.com/a/19593545.shtml   \n",
       "8                   数据挖掘工程师      https://www.liepin.com/a/19577517.shtml   \n",
       "9                      数据挖掘      https://www.liepin.com/a/19468911.shtml   \n",
       "10                   大数据挖掘师      https://www.liepin.com/a/19443397.shtml   \n",
       "11                高级数据挖掘工程师      https://www.liepin.com/a/19402695.shtml   \n",
       "12                资深数据挖掘工程师      https://www.liepin.com/a/19243319.shtml   \n",
       "13           网络安全大数据挖掘高级分析师      https://www.liepin.com/a/19208817.shtml   \n",
       "14               数据挖掘工程师/专家      https://www.liepin.com/a/18694767.shtml   \n",
       "15                资深数据挖掘工程师      https://www.liepin.com/a/18525023.shtml   \n",
       "16                   数据挖掘专家   https://www.liepin.com/job/197599882.shtml   \n",
       "17                资深数据挖掘工程师   https://www.liepin.com/job/194275684.shtml   \n",
       "18                风控数据挖掘工程师  https://www.liepin.com/job/1927229905.shtml   \n",
       "19               数据挖掘与分析工程师  https://www.liepin.com/job/1927212533.shtml   \n",
       "20                 大数据挖掘工程师  https://www.liepin.com/job/1927189899.shtml   \n",
       "21              （高级）数据挖掘工程师  https://www.liepin.com/job/1927160225.shtml   \n",
       "22             高级/资深数据挖掘工程师  https://www.liepin.com/job/1927098131.shtml   \n",
       "23                  数据挖掘工程师  https://www.liepin.com/job/1927094945.shtml   \n",
       "24  资深数据挖掘工程师-雷火用户体验研究中心001  https://www.liepin.com/job/1926571709.shtml   \n",
       "25           高级数据挖掘工程师-腾讯音乐  https://www.liepin.com/job/1926393875.shtml   \n",
       "26                  数据挖掘工程师  https://www.liepin.com/job/1925844529.shtml   \n",
       "27                  数据挖掘工程师  https://www.liepin.com/job/1925844527.shtml   \n",
       "28                  数据挖掘工程师  https://www.liepin.com/job/1925844525.shtml   \n",
       "29                  数据挖掘工程师  https://www.liepin.com/job/1925844521.shtml   \n",
       "30                  数据挖掘工程师  https://www.liepin.com/job/1925844519.shtml   \n",
       "31                  数据挖掘工程师  https://www.liepin.com/job/1925844517.shtml   \n",
       "32                  数据挖掘工程师  https://www.liepin.com/job/1925844515.shtml   \n",
       "33                  数据挖掘工程师  https://www.liepin.com/job/1925844513.shtml   \n",
       "34                  数据挖掘工程师  https://www.liepin.com/job/1925844511.shtml   \n",
       "35                  数据挖掘工程师  https://www.liepin.com/job/1925844509.shtml   \n",
       "36                  数据挖掘工程师  https://www.liepin.com/job/1925844507.shtml   \n",
       "37                  数据挖掘工程师  https://www.liepin.com/job/1925844505.shtml   \n",
       "38                  数据挖掘工程师  https://www.liepin.com/job/1925844503.shtml   \n",
       "39                  数据挖掘工程师  https://www.liepin.com/job/1925844501.shtml   \n",
       "\n",
       "                                        公司链接  \n",
       "0    https://www.liepin.com/company/9382271/  \n",
       "1    https://www.liepin.com/company/2036768/  \n",
       "2    https://www.liepin.com/company/1157027/  \n",
       "3    https://www.liepin.com/company/2036768/  \n",
       "4    https://www.liepin.com/company/2036768/  \n",
       "5    https://www.liepin.com/company/1157027/  \n",
       "6    https://www.liepin.com/company/2036768/  \n",
       "7                                             \n",
       "8                                             \n",
       "9                                             \n",
       "10                                            \n",
       "11                                            \n",
       "12                                            \n",
       "13                                            \n",
       "14                                            \n",
       "15                                            \n",
       "16   https://www.liepin.com/company/1157027/  \n",
       "17   https://www.liepin.com/company/1157027/  \n",
       "18  https://www.liepin.com/company/10204225/  \n",
       "19   https://www.liepin.com/company/9224435/  \n",
       "20   https://www.liepin.com/company/7885830/  \n",
       "21  https://www.liepin.com/company/10204225/  \n",
       "22   https://www.liepin.com/company/5964833/  \n",
       "23  https://www.liepin.com/company/10095707/  \n",
       "24   https://www.liepin.com/company/5964833/  \n",
       "25   https://www.liepin.com/company/7983148/  \n",
       "26  https://www.liepin.com/company/12146335/  \n",
       "27  https://www.liepin.com/company/12146335/  \n",
       "28  https://www.liepin.com/company/12146335/  \n",
       "29  https://www.liepin.com/company/12146335/  \n",
       "30  https://www.liepin.com/company/12146335/  \n",
       "31  https://www.liepin.com/company/12146335/  \n",
       "32  https://www.liepin.com/company/12146335/  \n",
       "33  https://www.liepin.com/company/12146335/  \n",
       "34  https://www.liepin.com/company/12146335/  \n",
       "35  https://www.liepin.com/company/12146335/  \n",
       "36  https://www.liepin.com/company/12146335/  \n",
       "37  https://www.liepin.com/company/12146335/  \n",
       "38  https://www.liepin.com/company/12146335/  \n",
       "39  https://www.liepin.com/company/12146335/  "
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "session= HTMLSession()\n",
    "\n",
    "#url=\"https://www.liepin.com/zhaopin/\"\n",
    "\n",
    "#爬取主函数\n",
    "def requests(url,params):\n",
    "    r= session.get(url,params = payload)\n",
    "    #设置主要的xpath路径\n",
    "    main= r.html.xpath(\"//div[contains(@class,'sojob-item-main')]\")\n",
    "    #使用字典存取特定的数据的位置\n",
    "    dict_xpath={\n",
    "        \"text\":{\n",
    "            \"职位\":\".//div[@class='job-info']/h3/a/text()\"\n",
    "        },\n",
    "        \"text_content\":{\n",
    "            \"时间\":\".//div[@class='job-info']/p[contains(@class,'time-info')]/time/@title\",\n",
    "\n",
    "            \"薪水\":\".//div[@class='job-info']/p/span[@class='text-warning']/text()\",\n",
    "            \"工作地点\":\".//p[contains(@class,condition)]/a/text()\",  #/div[@class='job-info']      [@class='area']\n",
    "            \"学历\":\".//span[@class='edu']/text()\",\n",
    "            \"经验\":\".//p[contains(@class,condition)]/span[3]/text()\",\n",
    "            \"公司名称\":\".//p[contains(@class,'company-name')]/a/text()\"\n",
    "        },\n",
    "        \"href\":{\n",
    "            \"职位链接\":\".//div[@class='job-info']/h3/a\",\n",
    "            \"公司链接\":\"//div[contains(@class,'sojob-item-main')]//p[@class='company-name']/a\" #/div[contains(@class,'company-info')]\n",
    "        }\n",
    "\n",
    "    }\n",
    "\n",
    "def get_e_text(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\"\".join([x.strip() for x in e.xpath(_xpath_)]) for e in main]\n",
    "        return(暂存结果)\n",
    "\n",
    "def get_text_content(_xpath_):\n",
    "        # 高级列表推导\n",
    "\n",
    "        暂存结果 =[ e.xpath(_xpath_)[0]  for e in main]\n",
    "\n",
    "        #暂存结果 =main#.xpath(_xpath_)\n",
    "        return(暂存结果)          \n",
    "\n",
    "    #如果为真，first=True返回第一个元素\n",
    "\n",
    "def get_href(_xpath_):          \n",
    "        暂存结果=[list(e.xpath(_xpath_)[0].absolute_links)[0]              \n",
    "                      if len(list(e.xpath(_xpath_)[0].absolute_links))>=1\n",
    "                            else \"\"  for e in main  ]\n",
    "\n",
    "\n",
    "        return(暂存结果)\n",
    "数据字典=dict()\n",
    "数据字典={k:get_text_content(v) for k,v in dict_xpath[\"text_content\"].items()}\n",
    "数据字典.update({k:get_e_text(v) for k,v in dict_xpath['text'].items()})\n",
    "数据字典.update({k:get_href(v) for k,v in dict_xpath['href'].items() })\n",
    "数据字典\n",
    "df=pd.DataFrame(数据字典)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 多个页面进行爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "session= HTMLSession()\n",
    "\n",
    "#url=\"https://www.liepin.com/zhaopin/\"\n",
    "\n",
    "#爬取主函数\n",
    "def requests(url,params):\n",
    "    r= session.get(url,params = payload)\n",
    "    #设置主要的xpath路径\n",
    "    main= r.html.xpath(\"//div[contains(@class,'sojob-item-main')]\")\n",
    "    #使用字典存取特定的数据的位置\n",
    "    dict_xpath={\n",
    "        \"text\":{\n",
    "            \"职位\":\".//div[@class='job-info']/h3/a/text()\"\n",
    "        },\n",
    "        \"text_content\":{\n",
    "            \"时间\":\".//div[@class='job-info']/p[contains(@class,'time-info')]/time/@title\",\n",
    "\n",
    "            \"薪水\":\".//div[@class='job-info']/p/span[@class='text-warning']/text()\",\n",
    "            \"工作地点\":\".//p[contains(@class,condition)]/a/text()\",  #/div[@class='job-info']      [@class='area']\n",
    "            \"学历\":\".//span[@class='edu']/text()\",\n",
    "            \"经验\":\".//p[contains(@class,condition)]/span[3]/text()\",\n",
    "            \"公司名称\":\".//p[contains(@class,'company-name')]/a/text()\"\n",
    "        },\n",
    "        \"href\":{\n",
    "            \"职位链接\":\".//div[@class='job-info']/h3/a\",\n",
    "            \"公司链接\":\"//div[contains(@class,'sojob-item-main')]//p[@class='company-name']/a\" #/div[contains(@class,'company-info')]\n",
    "        }\n",
    "\n",
    "    }\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "            # 高级列表推导\n",
    "            暂存结果 = [\"\".join([x.strip() for x in e.xpath(_xpath_)]) for e in main]\n",
    "            return(暂存结果)\n",
    "\n",
    "    def get_text_content(_xpath_):\n",
    "            # 高级列表推导\n",
    "\n",
    "            暂存结果 =[ e.xpath(_xpath_)[0]  for e in main]\n",
    "\n",
    "            #暂存结果 =main#.xpath(_xpath_)\n",
    "            return(暂存结果)          \n",
    "\n",
    "        #如果为真，first=True返回第一个元素\n",
    "\n",
    "    def get_href(_xpath_):          \n",
    "            暂存结果=[list(e.xpath(_xpath_)[0].absolute_links)[0]              \n",
    "                          if len(list(e.xpath(_xpath_)[0].absolute_links))>=1\n",
    "                                else \"\"  for e in main  ]\n",
    "\n",
    "\n",
    "            return(暂存结果)\n",
    "    数据字典=dict()\n",
    "    数据字典={k:get_text_content(v) for k,v in dict_xpath[\"text_content\"].items()}\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpath['text'].items()})\n",
    "    数据字典.update({k:get_href(v) for k,v in dict_xpath['href'].items() })\n",
    "    数据字典\n",
    "    df=pd.DataFrame(数据字典)\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>时间</th>\n",
       "      <th>薪水</th>\n",
       "      <th>工作地点</th>\n",
       "      <th>学历</th>\n",
       "      <th>经验</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>职位</th>\n",
       "      <th>职位链接</th>\n",
       "      <th>公司链接</th>\n",
       "      <th>热门公司类型</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>10-18k·12薪</td>\n",
       "      <td>广州</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>时代大数据</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1920605639.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9382271/</td>\n",
       "      <td>中国500强</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>数据中心-数据挖掘算法工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925789257.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "      <td>中国500强</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>10-15k·12薪</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>深演智能</td>\n",
       "      <td>2020校园招聘-数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1923453745.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1157027/</td>\n",
       "      <td>中国500强</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>上海-长宁区</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>点评App部-机器学习/数据挖掘算法专家</td>\n",
       "      <td>https://www.liepin.com/job/1923289363.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "      <td>中国500强</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>面议</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>美团点评</td>\n",
       "      <td>闪购-数据挖掘技术专家</td>\n",
       "      <td>https://www.liepin.com/job/1919432959.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2036768/</td>\n",
       "      <td>中国500强</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844509.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "      <td>上市公司</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>武汉</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844507.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "      <td>上市公司</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>上海</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844505.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "      <td>上市公司</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844503.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "      <td>上市公司</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>70-100k·12薪</td>\n",
       "      <td>青岛</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>卓尔人人</td>\n",
       "      <td>数据挖掘工程师</td>\n",
       "      <td>https://www.liepin.com/job/1925844501.shtml</td>\n",
       "      <td>https://www.liepin.com/company/12146335/</td>\n",
       "      <td>上市公司</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             时间           薪水    工作地点     学历     经验   公司名称  \\\n",
       "0   2020年04月03日   10-18k·12薪      广州  本科及以上   2年以上  时代大数据   \n",
       "1   2020年04月05日           面议    美团点评  本科及以上  本科及以上   美团点评   \n",
       "2   2020年04月05日   10-15k·12薪    深演智能   统招本科   统招本科   深演智能   \n",
       "3   2020年04月05日           面议  上海-长宁区  本科及以上   3年以上   美团点评   \n",
       "4   2020年04月05日           面议    美团点评  本科及以上  本科及以上   美团点评   \n",
       "..          ...          ...     ...    ...    ...    ...   \n",
       "35  2020年04月04日  70-100k·12薪      深圳   统招本科   5年以上   卓尔人人   \n",
       "36  2020年04月04日  70-100k·12薪      武汉   统招本科   5年以上   卓尔人人   \n",
       "37  2020年04月04日  70-100k·12薪      上海   统招本科   5年以上   卓尔人人   \n",
       "38  2020年04月04日  70-100k·12薪      北京   统招本科   5年以上   卓尔人人   \n",
       "39  2020年04月04日  70-100k·12薪      青岛   统招本科   5年以上   卓尔人人   \n",
       "\n",
       "                      职位                                         职位链接  \\\n",
       "0                数据挖掘工程师  https://www.liepin.com/job/1920605639.shtml   \n",
       "1         数据中心-数据挖掘算法工程师  https://www.liepin.com/job/1925789257.shtml   \n",
       "2       2020校园招聘-数据挖掘工程师  https://www.liepin.com/job/1923453745.shtml   \n",
       "3   点评App部-机器学习/数据挖掘算法专家  https://www.liepin.com/job/1923289363.shtml   \n",
       "4            闪购-数据挖掘技术专家  https://www.liepin.com/job/1919432959.shtml   \n",
       "..                   ...                                          ...   \n",
       "35               数据挖掘工程师  https://www.liepin.com/job/1925844509.shtml   \n",
       "36               数据挖掘工程师  https://www.liepin.com/job/1925844507.shtml   \n",
       "37               数据挖掘工程师  https://www.liepin.com/job/1925844505.shtml   \n",
       "38               数据挖掘工程师  https://www.liepin.com/job/1925844503.shtml   \n",
       "39               数据挖掘工程师  https://www.liepin.com/job/1925844501.shtml   \n",
       "\n",
       "                                        公司链接  热门公司类型  \n",
       "0    https://www.liepin.com/company/9382271/  中国500强  \n",
       "1    https://www.liepin.com/company/2036768/  中国500强  \n",
       "2    https://www.liepin.com/company/1157027/  中国500强  \n",
       "3    https://www.liepin.com/company/2036768/  中国500强  \n",
       "4    https://www.liepin.com/company/2036768/  中国500强  \n",
       "..                                       ...     ...  \n",
       "35  https://www.liepin.com/company/12146335/    上市公司  \n",
       "36  https://www.liepin.com/company/12146335/    上市公司  \n",
       "37  https://www.liepin.com/company/12146335/    上市公司  \n",
       "38  https://www.liepin.com/company/12146335/    上市公司  \n",
       "39  https://www.liepin.com/company/12146335/    上市公司  \n",
       "\n",
       "[240 rows x 10 columns]"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#   多个页面,即多个公司\n",
    "import pandas as pd\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "#payload = 参数_compTag_用户体验['中国500强']\n",
    "#DataFrame.assign(**kwargs)为DataFrame分配新列。返回一个新对象，该对象包含除新列之外的所有原始列。\n",
    "list_ef = list()\n",
    "#keyword为用户体验，key为数据，职位囊括上市公司，中国500强的公司的爬取\n",
    "for k,v in 参数_compTag_用户体验.items():     \n",
    "    payload = v\n",
    "    ef = requests( url, params = payload)#调用requests_liepin()函数\n",
    "    ef = df.assign(热门公司类型 = k)    \n",
    "    list_ef.append(ef)\n",
    "    \n",
    "数据挖掘 = pd.concat(list_ef)  #对数据的合并与重塑\n",
    "数据挖掘\n",
    "#payload\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 代码规整(重新练习)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立session及取特定url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "#建立会话\n",
    "from requests_html import HTMLSession\n",
    "session = HTMLSession()\n",
    "#url填指定要爬取的页面\n",
    "url=\"https://www.liepin.com/zhaopin/?industries=&subIndustry=&dqs=&salary=&jobKind=&pubTime=&compkind=&compscale=&industryType=&searchType=1&clean_condition=&isAnalysis=&init=1&sortFlag=15&flushckid=0&fromSearchBtn=1&headckid=52b1c0be834fc722&d_headId=41c0cceeae9b7e1f5aec9cc84ba87032&d_ckId=41c0cceeae9b7e1f5aec9cc84ba87032&d_sfrom=search_prime&d_curPage=0&d_pageSize=40&siTag=bFGQTbwE_AAQSb-u11jrBw%7EfA9rXquZc5IkJpXC-Ycixw&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98\"\n",
    "r = session.get( url )\n",
    "#获取url(筛选条件)\n",
    "all_url=r.html.xpath('//div[contains(@class,\"search-conditions\")]')[0]\\\n",
    "                .xpath('//dt[@class=\"search-title\"]/following-sibling::dd')[0]\\\n",
    "                .xpath('//dd[@class=\"comp-list\"]/ul/li')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'阿里巴巴': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=1072424&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '腾讯': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=7983148&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '字节跳动': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=7863078&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '美团点评': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=2036768&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '华为': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=954482&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '滴滴': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8076624&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " 'Baidu': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=884492&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " 'vivo': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=4787049&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '饿了么': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=2778803&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '深圳商道名家教育科技有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=9883043&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '小米': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=2174886&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '京东数字科技控股有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=215535&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '携程': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=182339&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '亚信科技': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=2399664&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " 'OPPO移动通信': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=3922321&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '京东商城': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=9747025&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '上海寻梦信息技术有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8537928&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '58同城': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=591850&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '深圳平安综合金融服务有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=7956861&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '网易集团': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=5964833&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '资采信息': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8830801&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '苏宁易购': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=2051657&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '上海钧正网络科技有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8515562&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '浩鲸云计算科技股份有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=7939633&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '猎聘招聘顾问团队': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=12176713&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '京东': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=1663745&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '蚂蚁金服(杭州)网络技术有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=9364743&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '北京中科特瑞科技有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=9497282&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '平安科技': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=1086536&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '便利蜂': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8983822&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '新浪微博': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=2638435&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '毕马威中国 KPMG China': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8169921&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '北京快手科技有限公司': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=8605497&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '中国移动通信有限公司研究院': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=9122701&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25',\n",
       " '好未来': '/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&compIds=922091&subIndustry=&industryType=&compscale=&key=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&init=-1&searchType=1&headckid=96731afb8363a6ab&flushckid=1&compkind=&fromSearchBtn=2&sortFlag=15&ckid=96731afb8363a6ab&jobKind=&industries=&clean_condition=&siTag=LiAE77uh7ygbLjiB5afMYg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=34ce54af34ecf6404adb0908733c7c25&d_curPage=0&d_pageSize=40&d_headId=34ce54af34ecf6404adb0908733c7c25'}"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_url={ x.xpath(\"a/span/text()\")[0]:x.xpath(\"a/@href\")[0] for x in all_url}\n",
    "company_url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用 urllib.parse 解析\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 检查相异值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scheme       1\n",
       "netloc       1\n",
       "path         1\n",
       "params       1\n",
       "query       35\n",
       "fragment     1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame([ urlparse(x) for x in company_url.values()])\n",
    "df.nunique()# 发现query不同 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "compIds          35\n",
       "key               1\n",
       "init              1\n",
       "searchType        1\n",
       "headckid          1\n",
       "flushckid         1\n",
       "fromSearchBtn     1\n",
       "sortFlag          1\n",
       "ckid              1\n",
       "siTag             1\n",
       "d_sfrom           1\n",
       "d_ckId            1\n",
       "d_curPage         1\n",
       "d_pageSize        1\n",
       "d_headId          1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#进一步检查query\n",
    "df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "df_qs.nunique()\n",
    "#发现compIds不同"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获得各公司专属ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'阿里巴巴': '1072424',\n",
       " '腾讯': '7983148',\n",
       " '字节跳动': '7863078',\n",
       " '美团点评': '2036768',\n",
       " '华为': '954482',\n",
       " '滴滴': '8076624',\n",
       " 'Baidu': '884492',\n",
       " 'vivo': '4787049',\n",
       " '饿了么': '2778803',\n",
       " '深圳商道名家教育科技有限公司': '9883043',\n",
       " '小米': '2174886',\n",
       " '京东数字科技控股有限公司': '215535',\n",
       " '携程': '182339',\n",
       " '亚信科技': '2399664',\n",
       " 'OPPO移动通信': '3922321',\n",
       " '京东商城': '9747025',\n",
       " '上海寻梦信息技术有限公司': '8537928',\n",
       " '58同城': '591850',\n",
       " '深圳平安综合金融服务有限公司': '7956861',\n",
       " '网易集团': '5964833',\n",
       " '资采信息': '8830801',\n",
       " '苏宁易购': '2051657',\n",
       " '上海钧正网络科技有限公司': '8515562',\n",
       " '浩鲸云计算科技股份有限公司': '7939633',\n",
       " '猎聘招聘顾问团队': '12176713',\n",
       " '京东': '1663745',\n",
       " '蚂蚁金服(杭州)网络技术有限公司': '9364743',\n",
       " '北京中科特瑞科技有限公司': '9497282',\n",
       " '平安科技': '1086536',\n",
       " '便利蜂': '8983822',\n",
       " '新浪微博': '2638435',\n",
       " '毕马威中国 KPMG China': '8169921',\n",
       " '北京快手科技有限公司': '8605497',\n",
       " '中国移动通信有限公司研究院': '9122701',\n",
       " '好未来': '922091'}"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from urllib.parse import urlparse, parse_qs\n",
    "#对url进行分解\n",
    "\n",
    "\n",
    "import pprint\n",
    "def parse_url_qs_for_compTag (url):\n",
    "    six_parts = urlparse(url)    \n",
    "    out = parse_qs(six_parts.query)\n",
    "    return (out)\n",
    "\n",
    "字典_compIds = { k:parse_url_qs_for_compTag(v)['compIds'][0] for k,v in company_url.items()}\n",
    "字典_compIds\n",
    "#df_qs\n",
    "        \n",
    "#e=list(company_url.values())\n",
    "#for x in e:\n",
    "#        pprint.pprint()\n",
    "#k= [ parse_url_qs_for_compTag(x)   for x in e   ]\n",
    "#k\n",
    "#df_qs.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###   参数模板生成"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 设置参数的值，例如：keyword、key等"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'58同城': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['591850'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " 'Baidu': {'ckid': ['970029fdeea53da8'],\n",
      "           'compIds': ['884492'],\n",
      "           'compTag': ['155'],\n",
      "           'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "           'd_curPage': ['0'],\n",
      "           'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "           'd_pageSize': ['40'],\n",
      "           'd_sfrom': ['search_prime'],\n",
      "           'flushckid': ['1'],\n",
      "           'fromSearchBtn': ['2'],\n",
      "           'headckid': ['970029fdeea53da8'],\n",
      "           'init': ['-1'],\n",
      "           'key': ['产品经理'],\n",
      "           'keyword': ['用户体验'],\n",
      "           'searchType': ['1'],\n",
      "           'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "           'sortFlag': ['15']},\n",
      " 'OPPO移动通信': {'ckid': ['970029fdeea53da8'],\n",
      "              'compIds': ['3922321'],\n",
      "              'compTag': ['155'],\n",
      "              'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "              'd_curPage': ['0'],\n",
      "              'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "              'd_pageSize': ['40'],\n",
      "              'd_sfrom': ['search_prime'],\n",
      "              'flushckid': ['1'],\n",
      "              'fromSearchBtn': ['2'],\n",
      "              'headckid': ['970029fdeea53da8'],\n",
      "              'init': ['-1'],\n",
      "              'key': ['产品经理'],\n",
      "              'keyword': ['用户体验'],\n",
      "              'searchType': ['1'],\n",
      "              'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "              'sortFlag': ['15']},\n",
      " 'vivo': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['4787049'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '上海寻梦信息技术有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                  'compIds': ['8537928'],\n",
      "                  'compTag': ['155'],\n",
      "                  'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_curPage': ['0'],\n",
      "                  'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_pageSize': ['40'],\n",
      "                  'd_sfrom': ['search_prime'],\n",
      "                  'flushckid': ['1'],\n",
      "                  'fromSearchBtn': ['2'],\n",
      "                  'headckid': ['970029fdeea53da8'],\n",
      "                  'init': ['-1'],\n",
      "                  'key': ['产品经理'],\n",
      "                  'keyword': ['用户体验'],\n",
      "                  'searchType': ['1'],\n",
      "                  'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                  'sortFlag': ['15']},\n",
      " '上海钧正网络科技有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                  'compIds': ['8515562'],\n",
      "                  'compTag': ['155'],\n",
      "                  'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_curPage': ['0'],\n",
      "                  'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_pageSize': ['40'],\n",
      "                  'd_sfrom': ['search_prime'],\n",
      "                  'flushckid': ['1'],\n",
      "                  'fromSearchBtn': ['2'],\n",
      "                  'headckid': ['970029fdeea53da8'],\n",
      "                  'init': ['-1'],\n",
      "                  'key': ['产品经理'],\n",
      "                  'keyword': ['用户体验'],\n",
      "                  'searchType': ['1'],\n",
      "                  'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                  'sortFlag': ['15']},\n",
      " '中国移动通信有限公司研究院': {'ckid': ['970029fdeea53da8'],\n",
      "                   'compIds': ['9122701'],\n",
      "                   'compTag': ['155'],\n",
      "                   'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                   'd_curPage': ['0'],\n",
      "                   'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                   'd_pageSize': ['40'],\n",
      "                   'd_sfrom': ['search_prime'],\n",
      "                   'flushckid': ['1'],\n",
      "                   'fromSearchBtn': ['2'],\n",
      "                   'headckid': ['970029fdeea53da8'],\n",
      "                   'init': ['-1'],\n",
      "                   'key': ['产品经理'],\n",
      "                   'keyword': ['用户体验'],\n",
      "                   'searchType': ['1'],\n",
      "                   'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                   'sortFlag': ['15']},\n",
      " '亚信科技': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['2399664'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '京东': {'ckid': ['970029fdeea53da8'],\n",
      "        'compIds': ['1663745'],\n",
      "        'compTag': ['155'],\n",
      "        'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_curPage': ['0'],\n",
      "        'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_pageSize': ['40'],\n",
      "        'd_sfrom': ['search_prime'],\n",
      "        'flushckid': ['1'],\n",
      "        'fromSearchBtn': ['2'],\n",
      "        'headckid': ['970029fdeea53da8'],\n",
      "        'init': ['-1'],\n",
      "        'key': ['产品经理'],\n",
      "        'keyword': ['用户体验'],\n",
      "        'searchType': ['1'],\n",
      "        'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "        'sortFlag': ['15']},\n",
      " '京东商城': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['9747025'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '京东数字科技控股有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                  'compIds': ['215535'],\n",
      "                  'compTag': ['155'],\n",
      "                  'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_curPage': ['0'],\n",
      "                  'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_pageSize': ['40'],\n",
      "                  'd_sfrom': ['search_prime'],\n",
      "                  'flushckid': ['1'],\n",
      "                  'fromSearchBtn': ['2'],\n",
      "                  'headckid': ['970029fdeea53da8'],\n",
      "                  'init': ['-1'],\n",
      "                  'key': ['产品经理'],\n",
      "                  'keyword': ['用户体验'],\n",
      "                  'searchType': ['1'],\n",
      "                  'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                  'sortFlag': ['15']},\n",
      " '便利蜂': {'ckid': ['970029fdeea53da8'],\n",
      "         'compIds': ['8983822'],\n",
      "         'compTag': ['155'],\n",
      "         'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_curPage': ['0'],\n",
      "         'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_pageSize': ['40'],\n",
      "         'd_sfrom': ['search_prime'],\n",
      "         'flushckid': ['1'],\n",
      "         'fromSearchBtn': ['2'],\n",
      "         'headckid': ['970029fdeea53da8'],\n",
      "         'init': ['-1'],\n",
      "         'key': ['产品经理'],\n",
      "         'keyword': ['用户体验'],\n",
      "         'searchType': ['1'],\n",
      "         'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "         'sortFlag': ['15']},\n",
      " '北京中科特瑞科技有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                  'compIds': ['9497282'],\n",
      "                  'compTag': ['155'],\n",
      "                  'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_curPage': ['0'],\n",
      "                  'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                  'd_pageSize': ['40'],\n",
      "                  'd_sfrom': ['search_prime'],\n",
      "                  'flushckid': ['1'],\n",
      "                  'fromSearchBtn': ['2'],\n",
      "                  'headckid': ['970029fdeea53da8'],\n",
      "                  'init': ['-1'],\n",
      "                  'key': ['产品经理'],\n",
      "                  'keyword': ['用户体验'],\n",
      "                  'searchType': ['1'],\n",
      "                  'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                  'sortFlag': ['15']},\n",
      " '北京快手科技有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                'compIds': ['8605497'],\n",
      "                'compTag': ['155'],\n",
      "                'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                'd_curPage': ['0'],\n",
      "                'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                'd_pageSize': ['40'],\n",
      "                'd_sfrom': ['search_prime'],\n",
      "                'flushckid': ['1'],\n",
      "                'fromSearchBtn': ['2'],\n",
      "                'headckid': ['970029fdeea53da8'],\n",
      "                'init': ['-1'],\n",
      "                'key': ['产品经理'],\n",
      "                'keyword': ['用户体验'],\n",
      "                'searchType': ['1'],\n",
      "                'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                'sortFlag': ['15']},\n",
      " '华为': {'ckid': ['970029fdeea53da8'],\n",
      "        'compIds': ['954482'],\n",
      "        'compTag': ['155'],\n",
      "        'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_curPage': ['0'],\n",
      "        'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_pageSize': ['40'],\n",
      "        'd_sfrom': ['search_prime'],\n",
      "        'flushckid': ['1'],\n",
      "        'fromSearchBtn': ['2'],\n",
      "        'headckid': ['970029fdeea53da8'],\n",
      "        'init': ['-1'],\n",
      "        'key': ['产品经理'],\n",
      "        'keyword': ['用户体验'],\n",
      "        'searchType': ['1'],\n",
      "        'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "        'sortFlag': ['15']},\n",
      " '好未来': {'ckid': ['970029fdeea53da8'],\n",
      "         'compIds': ['922091'],\n",
      "         'compTag': ['155'],\n",
      "         'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_curPage': ['0'],\n",
      "         'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_pageSize': ['40'],\n",
      "         'd_sfrom': ['search_prime'],\n",
      "         'flushckid': ['1'],\n",
      "         'fromSearchBtn': ['2'],\n",
      "         'headckid': ['970029fdeea53da8'],\n",
      "         'init': ['-1'],\n",
      "         'key': ['产品经理'],\n",
      "         'keyword': ['用户体验'],\n",
      "         'searchType': ['1'],\n",
      "         'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "         'sortFlag': ['15']},\n",
      " '字节跳动': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['7863078'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '小米': {'ckid': ['970029fdeea53da8'],\n",
      "        'compIds': ['2174886'],\n",
      "        'compTag': ['155'],\n",
      "        'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_curPage': ['0'],\n",
      "        'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_pageSize': ['40'],\n",
      "        'd_sfrom': ['search_prime'],\n",
      "        'flushckid': ['1'],\n",
      "        'fromSearchBtn': ['2'],\n",
      "        'headckid': ['970029fdeea53da8'],\n",
      "        'init': ['-1'],\n",
      "        'key': ['产品经理'],\n",
      "        'keyword': ['用户体验'],\n",
      "        'searchType': ['1'],\n",
      "        'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "        'sortFlag': ['15']},\n",
      " '平安科技': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['1086536'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '携程': {'ckid': ['970029fdeea53da8'],\n",
      "        'compIds': ['182339'],\n",
      "        'compTag': ['155'],\n",
      "        'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_curPage': ['0'],\n",
      "        'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_pageSize': ['40'],\n",
      "        'd_sfrom': ['search_prime'],\n",
      "        'flushckid': ['1'],\n",
      "        'fromSearchBtn': ['2'],\n",
      "        'headckid': ['970029fdeea53da8'],\n",
      "        'init': ['-1'],\n",
      "        'key': ['产品经理'],\n",
      "        'keyword': ['用户体验'],\n",
      "        'searchType': ['1'],\n",
      "        'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "        'sortFlag': ['15']},\n",
      " '新浪微博': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['2638435'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '毕马威中国 KPMG China': {'ckid': ['970029fdeea53da8'],\n",
      "                      'compIds': ['8169921'],\n",
      "                      'compTag': ['155'],\n",
      "                      'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                      'd_curPage': ['0'],\n",
      "                      'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                      'd_pageSize': ['40'],\n",
      "                      'd_sfrom': ['search_prime'],\n",
      "                      'flushckid': ['1'],\n",
      "                      'fromSearchBtn': ['2'],\n",
      "                      'headckid': ['970029fdeea53da8'],\n",
      "                      'init': ['-1'],\n",
      "                      'key': ['产品经理'],\n",
      "                      'keyword': ['用户体验'],\n",
      "                      'searchType': ['1'],\n",
      "                      'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                      'sortFlag': ['15']},\n",
      " '浩鲸云计算科技股份有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                   'compIds': ['7939633'],\n",
      "                   'compTag': ['155'],\n",
      "                   'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                   'd_curPage': ['0'],\n",
      "                   'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                   'd_pageSize': ['40'],\n",
      "                   'd_sfrom': ['search_prime'],\n",
      "                   'flushckid': ['1'],\n",
      "                   'fromSearchBtn': ['2'],\n",
      "                   'headckid': ['970029fdeea53da8'],\n",
      "                   'init': ['-1'],\n",
      "                   'key': ['产品经理'],\n",
      "                   'keyword': ['用户体验'],\n",
      "                   'searchType': ['1'],\n",
      "                   'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                   'sortFlag': ['15']},\n",
      " '深圳商道名家教育科技有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                    'compIds': ['9883043'],\n",
      "                    'compTag': ['155'],\n",
      "                    'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                    'd_curPage': ['0'],\n",
      "                    'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                    'd_pageSize': ['40'],\n",
      "                    'd_sfrom': ['search_prime'],\n",
      "                    'flushckid': ['1'],\n",
      "                    'fromSearchBtn': ['2'],\n",
      "                    'headckid': ['970029fdeea53da8'],\n",
      "                    'init': ['-1'],\n",
      "                    'key': ['产品经理'],\n",
      "                    'keyword': ['用户体验'],\n",
      "                    'searchType': ['1'],\n",
      "                    'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                    'sortFlag': ['15']},\n",
      " '深圳平安综合金融服务有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                    'compIds': ['7956861'],\n",
      "                    'compTag': ['155'],\n",
      "                    'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                    'd_curPage': ['0'],\n",
      "                    'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                    'd_pageSize': ['40'],\n",
      "                    'd_sfrom': ['search_prime'],\n",
      "                    'flushckid': ['1'],\n",
      "                    'fromSearchBtn': ['2'],\n",
      "                    'headckid': ['970029fdeea53da8'],\n",
      "                    'init': ['-1'],\n",
      "                    'key': ['产品经理'],\n",
      "                    'keyword': ['用户体验'],\n",
      "                    'searchType': ['1'],\n",
      "                    'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                    'sortFlag': ['15']},\n",
      " '滴滴': {'ckid': ['970029fdeea53da8'],\n",
      "        'compIds': ['8076624'],\n",
      "        'compTag': ['155'],\n",
      "        'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_curPage': ['0'],\n",
      "        'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_pageSize': ['40'],\n",
      "        'd_sfrom': ['search_prime'],\n",
      "        'flushckid': ['1'],\n",
      "        'fromSearchBtn': ['2'],\n",
      "        'headckid': ['970029fdeea53da8'],\n",
      "        'init': ['-1'],\n",
      "        'key': ['产品经理'],\n",
      "        'keyword': ['用户体验'],\n",
      "        'searchType': ['1'],\n",
      "        'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "        'sortFlag': ['15']},\n",
      " '猎聘招聘顾问团队': {'ckid': ['970029fdeea53da8'],\n",
      "              'compIds': ['12176713'],\n",
      "              'compTag': ['155'],\n",
      "              'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "              'd_curPage': ['0'],\n",
      "              'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "              'd_pageSize': ['40'],\n",
      "              'd_sfrom': ['search_prime'],\n",
      "              'flushckid': ['1'],\n",
      "              'fromSearchBtn': ['2'],\n",
      "              'headckid': ['970029fdeea53da8'],\n",
      "              'init': ['-1'],\n",
      "              'key': ['产品经理'],\n",
      "              'keyword': ['用户体验'],\n",
      "              'searchType': ['1'],\n",
      "              'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "              'sortFlag': ['15']},\n",
      " '网易集团': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['5964833'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '美团点评': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['2036768'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '腾讯': {'ckid': ['970029fdeea53da8'],\n",
      "        'compIds': ['7983148'],\n",
      "        'compTag': ['155'],\n",
      "        'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_curPage': ['0'],\n",
      "        'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "        'd_pageSize': ['40'],\n",
      "        'd_sfrom': ['search_prime'],\n",
      "        'flushckid': ['1'],\n",
      "        'fromSearchBtn': ['2'],\n",
      "        'headckid': ['970029fdeea53da8'],\n",
      "        'init': ['-1'],\n",
      "        'key': ['产品经理'],\n",
      "        'keyword': ['用户体验'],\n",
      "        'searchType': ['1'],\n",
      "        'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "        'sortFlag': ['15']},\n",
      " '苏宁易购': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['2051657'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '蚂蚁金服(杭州)网络技术有限公司': {'ckid': ['970029fdeea53da8'],\n",
      "                      'compIds': ['9364743'],\n",
      "                      'compTag': ['155'],\n",
      "                      'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                      'd_curPage': ['0'],\n",
      "                      'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "                      'd_pageSize': ['40'],\n",
      "                      'd_sfrom': ['search_prime'],\n",
      "                      'flushckid': ['1'],\n",
      "                      'fromSearchBtn': ['2'],\n",
      "                      'headckid': ['970029fdeea53da8'],\n",
      "                      'init': ['-1'],\n",
      "                      'key': ['产品经理'],\n",
      "                      'keyword': ['用户体验'],\n",
      "                      'searchType': ['1'],\n",
      "                      'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "                      'sortFlag': ['15']},\n",
      " '资采信息': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['8830801'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '阿里巴巴': {'ckid': ['970029fdeea53da8'],\n",
      "          'compIds': ['1072424'],\n",
      "          'compTag': ['155'],\n",
      "          'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_curPage': ['0'],\n",
      "          'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "          'd_pageSize': ['40'],\n",
      "          'd_sfrom': ['search_prime'],\n",
      "          'flushckid': ['1'],\n",
      "          'fromSearchBtn': ['2'],\n",
      "          'headckid': ['970029fdeea53da8'],\n",
      "          'init': ['-1'],\n",
      "          'key': ['产品经理'],\n",
      "          'keyword': ['用户体验'],\n",
      "          'searchType': ['1'],\n",
      "          'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "          'sortFlag': ['15']},\n",
      " '饿了么': {'ckid': ['970029fdeea53da8'],\n",
      "         'compIds': ['2778803'],\n",
      "         'compTag': ['155'],\n",
      "         'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_curPage': ['0'],\n",
      "         'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'],\n",
      "         'd_pageSize': ['40'],\n",
      "         'd_sfrom': ['search_prime'],\n",
      "         'flushckid': ['1'],\n",
      "         'fromSearchBtn': ['2'],\n",
      "         'headckid': ['970029fdeea53da8'],\n",
      "         'init': ['-1'],\n",
      "         'key': ['产品经理'],\n",
      "         'keyword': ['用户体验'],\n",
      "         'searchType': ['1'],\n",
      "         'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'],\n",
      "         'sortFlag': ['15']}}\n"
     ]
    }
   ],
   "source": [
    "def 参数模板生成(compIds , keyword,key ):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['compIds'] = compIds\n",
    "    参数['keyword'] = keyword\n",
    "    参数['key']=key\n",
    "    return (参数)\n",
    "\n",
    "参数_配置url筛选 = { k:参数模板生成(compIds = [v],keyword = ['用户体验'],key=['产品经理']) for k,v in 字典_compIds.items()}\n",
    "pprint.pprint(参数_配置url筛选)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 配置请求url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key': ['数据分析'], 'init': ['-1'], 'searchType': ['1'], 'headckid': ['970029fdeea53da8'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'compTag': ['155'], 'sortFlag': ['15'], 'ckid': ['970029fdeea53da8'], 'siTag': ['LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_prime'], 'd_ckId': ['5b217fbd8e34ad6a68b83bd1bebc133f'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['5b217fbd8e34ad6a68b83bd1bebc133f'], 'compIds': ['1663745'], 'keyword': ['python']}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://www.liepin.com/zhaopin/?key=%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90&init=-1&searchType=1&headckid=970029fdeea53da8&flushckid=1&fromSearchBtn=2&compTag=155&sortFlag=15&ckid=970029fdeea53da8&siTag=LiAE77uh7ygbLjiB5afMYg~fA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=5b217fbd8e34ad6a68b83bd1bebc133f&d_curPage=0&d_pageSize=40&d_headId=5b217fbd8e34ad6a68b83bd1bebc133f&compIds=1663745&keyword=python'"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#多个页面准备测试1 世界500强\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "session = HTMLSession()\n",
    "payload = 参数_配置url筛选['京东']\n",
    "print(payload)\n",
    "r = session.get( url, params = payload)\n",
    "r.url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 爬取信息函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "session= HTMLSession()\n",
    "\n",
    "#url=\"https://www.liepin.com/zhaopin/\"\n",
    "\n",
    "#爬取主函数\n",
    "def requests(url,params):\n",
    "    r= session.get(url,params = payload)\n",
    "    #设置主要的xpath路径\n",
    "    main= r.html.xpath(\"//div[contains(@class,'sojob-item-main')]\")\n",
    "    #使用字典存取特定的数据的位置\n",
    "    dict_xpath={\n",
    "        \"text\":{\n",
    "            \"职位\":\".//div[@class='job-info']/h3/a/text()\"\n",
    "        },\n",
    "        \"text_content\":{\n",
    "            \"时间\":\".//div[@class='job-info']/p[contains(@class,'time-info')]/time/@title\",\n",
    "\n",
    "            \"薪水\":\".//div[@class='job-info']/p/span[@class='text-warning']/text()\",\n",
    "            \"工作地点\":\".//p[contains(@class,condition)]/a/text()\",  #/div[@class='job-info']      [@class='area']\n",
    "            \"学历\":\".//span[@class='edu']/text()\",\n",
    "            \"经验\":\".//p[contains(@class,condition)]/span[3]/text()\",\n",
    "            \"公司名称\":\".//p[contains(@class,'company-name')]/a/text()\"\n",
    "        },\n",
    "        \"href\":{\n",
    "            \"职位链接\":\".//div[@class='job-info']/h3/a\",\n",
    "            \"公司链接\":\"//div[contains(@class,'sojob-item-main')]//p[@class='company-name']/a\" #/div[contains(@class,'company-info')]\n",
    "        }\n",
    "\n",
    "    }\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "            # 高级列表推导\n",
    "            暂存结果 = [\"\".join([x.strip() for x in e.xpath(_xpath_)]) for e in main]\n",
    "            return(暂存结果)\n",
    "\n",
    "    def get_text_content(_xpath_):\n",
    "            # 高级列表推导\n",
    "\n",
    "            暂存结果 =[ e.xpath(_xpath_)[0]  for e in main]\n",
    "\n",
    "            #暂存结果 =main#.xpath(_xpath_)\n",
    "            return(暂存结果)          \n",
    "\n",
    "        #如果为真，first=True返回第一个元素\n",
    "\n",
    "    def get_href(_xpath_):          \n",
    "            暂存结果=[list(e.xpath(_xpath_)[0].absolute_links)[0]              \n",
    "                          if len(list(e.xpath(_xpath_)[0].absolute_links))>=1\n",
    "                                else \"\"  for e in main  ]\n",
    "\n",
    "\n",
    "            return(暂存结果)\n",
    "    数据字典=dict()\n",
    "    数据字典={k:get_text_content(v) for k,v in dict_xpath[\"text_content\"].items()}\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpath['text'].items()})\n",
    "    数据字典.update({k:get_href(v) for k,v in dict_xpath['href'].items() })\n",
    "    数据字典\n",
    "    df=pd.DataFrame(数据字典)\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 请求URL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####   多个页面,即多个公司"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "#配置请求筛选要求\n",
    "def 参数模板生成(compIds , keyword,key ):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['compIds'] = compIds\n",
    "    参数['keyword'] = keyword\n",
    "    参数['key']=key\n",
    "    return (参数)\n",
    "\n",
    "参数_配置url筛选 = { k:参数模板生成(compIds = [v], keyword = ['用户体验'],key=['交互设计师']) for k,v in 字典_compIds.items()}\n",
    "#pprint.pprint(参数_配置url筛选)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>时间</th>\n",
       "      <th>薪水</th>\n",
       "      <th>工作地点</th>\n",
       "      <th>学历</th>\n",
       "      <th>经验</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>职位</th>\n",
       "      <th>职位链接</th>\n",
       "      <th>公司链接</th>\n",
       "      <th>热门公司类型</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>阿里巴巴</td>\n",
       "      <td>新零售技术事业群-数据分析专家-躺平算法-淘系技术部</td>\n",
       "      <td>https://www.liepin.com/job/1927207743.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1072424/</td>\n",
       "      <td>阿里巴巴</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>阿里巴巴</td>\n",
       "      <td>阿里云智能事业群-销售数据分析专家-杭州</td>\n",
       "      <td>https://www.liepin.com/job/1927156871.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1072424/</td>\n",
       "      <td>阿里巴巴</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>阿里巴巴</td>\n",
       "      <td>新零售技术事业群-数据分析专家-淘系技术部</td>\n",
       "      <td>https://www.liepin.com/job/1921429517.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1072424/</td>\n",
       "      <td>阿里巴巴</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>阿里巴巴</td>\n",
       "      <td>集团公共事务部-数据分析专家-杭州</td>\n",
       "      <td>https://www.liepin.com/job/1927153713.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1072424/</td>\n",
       "      <td>阿里巴巴</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>杭州</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>阿里巴巴</td>\n",
       "      <td>集团公共事务部-数据分析专家-北京</td>\n",
       "      <td>https://www.liepin.com/job/1927153709.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1072424/</td>\n",
       "      <td>阿里巴巴</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019年06月23日</td>\n",
       "      <td>12-18k·12薪</td>\n",
       "      <td>南京</td>\n",
       "      <td>大专及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>苏宁易购</td>\n",
       "      <td>UE设计师</td>\n",
       "      <td>https://www.liepin.com/job/1920933303.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2051657/</td>\n",
       "      <td>苏宁易购</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2019年09月20日</td>\n",
       "      <td>8-16k·12薪</td>\n",
       "      <td>南京-栖霞区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>苏宁易购</td>\n",
       "      <td>交互设计师（智能硬件）</td>\n",
       "      <td>https://www.liepin.com/job/1920097181.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2051657/</td>\n",
       "      <td>苏宁易购</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2019年09月20日</td>\n",
       "      <td>15-20k·12薪</td>\n",
       "      <td>苏宁易购</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>苏宁易购</td>\n",
       "      <td>小程序前端开发工程师</td>\n",
       "      <td>https://www.liepin.com/job/1921761173.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2051657/</td>\n",
       "      <td>苏宁易购</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2019年04月02日</td>\n",
       "      <td>45-70k·14薪</td>\n",
       "      <td>南京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>8年以上</td>\n",
       "      <td>苏宁易购</td>\n",
       "      <td>产品总监（广场方向）</td>\n",
       "      <td>https://www.liepin.com/job/1918756095.shtml</td>\n",
       "      <td>https://www.liepin.com/company/2051657/</td>\n",
       "      <td>苏宁易购</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019年02月26日</td>\n",
       "      <td>15-30k·12薪</td>\n",
       "      <td>北京</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>京东</td>\n",
       "      <td>用户体验师</td>\n",
       "      <td>https://www.liepin.com/job/1916928049.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1663745/</td>\n",
       "      <td>京东</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1165 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             时间          薪水    工作地点     学历    经验  公司名称  \\\n",
       "0   2020年04月03日          面议      杭州   学历不限  3年以上  阿里巴巴   \n",
       "1   2020年04月03日          面议      杭州   学历不限  5年以上  阿里巴巴   \n",
       "2   2020年04月03日          面议      杭州   学历不限  3年以上  阿里巴巴   \n",
       "3   2020年04月02日          面议      杭州   学历不限  5年以上  阿里巴巴   \n",
       "4   2020年04月02日          面议      杭州   学历不限  5年以上  阿里巴巴   \n",
       "..          ...         ...     ...    ...   ...   ...   \n",
       "4   2019年06月23日  12-18k·12薪      南京  大专及以上  5年以上  苏宁易购   \n",
       "5   2019年09月20日   8-16k·12薪  南京-栖霞区   统招本科  2年以上  苏宁易购   \n",
       "6   2019年09月20日  15-20k·12薪    苏宁易购   统招本科  统招本科  苏宁易购   \n",
       "7   2019年04月02日  45-70k·14薪      南京  本科及以上  8年以上  苏宁易购   \n",
       "0   2019年02月26日  15-30k·12薪      北京   统招本科  5年以上    京东   \n",
       "\n",
       "                            职位                                         职位链接  \\\n",
       "0   新零售技术事业群-数据分析专家-躺平算法-淘系技术部  https://www.liepin.com/job/1927207743.shtml   \n",
       "1         阿里云智能事业群-销售数据分析专家-杭州  https://www.liepin.com/job/1927156871.shtml   \n",
       "2        新零售技术事业群-数据分析专家-淘系技术部  https://www.liepin.com/job/1921429517.shtml   \n",
       "3            集团公共事务部-数据分析专家-杭州  https://www.liepin.com/job/1927153713.shtml   \n",
       "4            集团公共事务部-数据分析专家-北京  https://www.liepin.com/job/1927153709.shtml   \n",
       "..                         ...                                          ...   \n",
       "4                        UE设计师  https://www.liepin.com/job/1920933303.shtml   \n",
       "5                  交互设计师（智能硬件）  https://www.liepin.com/job/1920097181.shtml   \n",
       "6                   小程序前端开发工程师  https://www.liepin.com/job/1921761173.shtml   \n",
       "7                   产品总监（广场方向）  https://www.liepin.com/job/1918756095.shtml   \n",
       "0                        用户体验师  https://www.liepin.com/job/1916928049.shtml   \n",
       "\n",
       "                                       公司链接 热门公司类型  \n",
       "0   https://www.liepin.com/company/1072424/   阿里巴巴  \n",
       "1   https://www.liepin.com/company/1072424/   阿里巴巴  \n",
       "2   https://www.liepin.com/company/1072424/   阿里巴巴  \n",
       "3   https://www.liepin.com/company/1072424/   阿里巴巴  \n",
       "4   https://www.liepin.com/company/1072424/   阿里巴巴  \n",
       "..                                      ...    ...  \n",
       "4   https://www.liepin.com/company/2051657/   苏宁易购  \n",
       "5   https://www.liepin.com/company/2051657/   苏宁易购  \n",
       "6   https://www.liepin.com/company/2051657/   苏宁易购  \n",
       "7   https://www.liepin.com/company/2051657/   苏宁易购  \n",
       "0   https://www.liepin.com/company/1663745/     京东  \n",
       "\n",
       "[1165 rows x 10 columns]"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#   多个页面,即多个公司\n",
    "import pandas as pd\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "#payload = 参数_配置url筛选['阿里巴巴']\n",
    "#DataFrame.assign(**kwargs)为DataFrame分配新列。返回一个新对象，该对象包含除新列之外的所有原始列。\n",
    "list_ef = list()\n",
    "#keyword为用户体验，key为数据，职位囊括上市公司，中国500强的公司的爬取\n",
    "for k,v in 参数_配置url筛选.items():     \n",
    "    payload = v\n",
    "    df = requests( url, params = payload)#调用requests_liepin()函数\n",
    "    df = df.assign(热门公司类型 = k)    \n",
    "    list_df.append(df)\n",
    "    \n",
    "df_all_message = pd.concat(list_df)  #对数据的合并与重塑\n",
    "df_all_message "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 单个页面个页面,即指定公司"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "#配置请求筛选要求\n",
    "def 参数模板生成(compIds , keyword,key ):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['compIds'] = compIds\n",
    "    参数['keyword'] = keyword\n",
    "    参数['key']=key\n",
    "    return (参数)\n",
    "\n",
    "参数_配置url筛选 = { k:参数模板生成(compIds = [v], keyword = ['PRD'],key=['产品经理']) for k,v in 字典_compIds.items()}\n",
    "#pprint.pprint(参数_配置url筛选)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>时间</th>\n",
       "      <th>薪水</th>\n",
       "      <th>工作地点</th>\n",
       "      <th>学历</th>\n",
       "      <th>经验</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>职位</th>\n",
       "      <th>职位链接</th>\n",
       "      <th>公司链接</th>\n",
       "      <th>热门公司类型</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020年04月05日</td>\n",
       "      <td>25-40k·12薪</td>\n",
       "      <td>深圳-南山区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>腾讯云生态产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927237017.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020年04月04日</td>\n",
       "      <td>30-60k·12薪</td>\n",
       "      <td>深圳-南山区</td>\n",
       "      <td>统招本科</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>BI数据产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1926393923.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020年04月03日</td>\n",
       "      <td>35-55k·16薪</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>10年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1926042901.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>CSIG16-垂类信息服务建设产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927177593.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>30628-腾讯广告行业服务高级产品经理（北京）</td>\n",
       "      <td>https://www.liepin.com/job/1927177589.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>31967-企鹅电竞用户增长渠道产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927177563.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>32272-腾讯视频TV产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927177551.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>36960-数据产品经理（北京或深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927177539.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>上海</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>30360-数据增长产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927177477.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>上海</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>19332-产品运营经理（上海）</td>\n",
       "      <td>https://www.liepin.com/job/1927177475.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>23674-腾讯新闻信息流体验产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927177473.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>21309-在线教育--企鹅辅导AI产品策划经理</td>\n",
       "      <td>https://www.liepin.com/job/1927141655.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>18427-理财通数据产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927141635.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>上海</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>30628-腾讯广告数据产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927141599.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>20052-搜索商业化产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927141595.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>59577-29912-生态工具产品经理（北京）</td>\n",
       "      <td>https://www.liepin.com/job/1927141575.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>36242-金融产品风险政策经理/高级经理</td>\n",
       "      <td>https://www.liepin.com/job/1927130685.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>36242-金融产品风险管理经理/高级经理</td>\n",
       "      <td>https://www.liepin.com/job/1927130683.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>36242-金融产品风险监控经理/高级经理</td>\n",
       "      <td>https://www.liepin.com/job/1927130681.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>34975-高级海外产品研究经理</td>\n",
       "      <td>https://www.liepin.com/job/1927130311.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>TME-QQ音乐数据产品高级经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927100431.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>59545-30632-腾讯广告商业产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927100425.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>1年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>23518-社交产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927100355.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>25667-电销系统产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927100333.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>经验不限</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>22989-物联网生态平台产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927038135.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>经验不限</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>22989-智慧城市物联网中台产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927038133.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>PCG22-创新项目数据中台产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038125.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>25667-腾讯云合作产品交付经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038111.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>59431-26699-智慧零售数据中心高级产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927038103.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>28481-医疗行业产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038063.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>59410-22989-腾讯云经管平台公共应用领域高级产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927038059.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>27092-联运游戏商业策划产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038041.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>30360-社交用户增长数据产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038039.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>WXG04-小游戏平台策略产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038037.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>22989-视频云产品经理（深圳）</td>\n",
       "      <td>https://www.liepin.com/job/1927038031.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>22989-CDN高级产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927010835.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>22989-腾云扣钉产品经理(全资公司编制)</td>\n",
       "      <td>https://www.liepin.com/job/1927010833.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>59381-22989-腾云扣钉产品运营经理(全资公司编制)</td>\n",
       "      <td>https://www.liepin.com/job/1927010831.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>深圳</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>28606-微信支付财税行业产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927010769.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2020年04月02日</td>\n",
       "      <td>面议</td>\n",
       "      <td>北京</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>2年以上</td>\n",
       "      <td>腾讯</td>\n",
       "      <td>20052-用户评估产品经理</td>\n",
       "      <td>https://www.liepin.com/job/1927010763.shtml</td>\n",
       "      <td>https://www.liepin.com/company/7983148/</td>\n",
       "      <td>好未来</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             时间          薪水    工作地点     学历     经验 公司名称  \\\n",
       "0   2020年04月05日  25-40k·12薪  深圳-南山区   统招本科   5年以上   腾讯   \n",
       "1   2020年04月04日  30-60k·12薪  深圳-南山区   统招本科   5年以上   腾讯   \n",
       "2   2020年04月03日  35-55k·16薪      深圳  本科及以上  10年以上   腾讯   \n",
       "3   2020年04月02日          面议      北京  本科及以上   5年以上   腾讯   \n",
       "4   2020年04月02日          面议      北京  本科及以上   5年以上   腾讯   \n",
       "5   2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "6   2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "7   2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "8   2020年04月02日          面议      上海  本科及以上   2年以上   腾讯   \n",
       "9   2020年04月02日          面议      上海  本科及以上   5年以上   腾讯   \n",
       "10  2020年04月02日          面议      北京  本科及以上   3年以上   腾讯   \n",
       "11  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "12  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "13  2020年04月02日          面议      上海   学历不限   3年以上   腾讯   \n",
       "14  2020年04月02日          面议      北京  本科及以上   3年以上   腾讯   \n",
       "15  2020年04月02日          面议      北京  本科及以上   3年以上   腾讯   \n",
       "16  2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "17  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "18  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "19  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "20  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "21  2020年04月02日          面议      北京  本科及以上   3年以上   腾讯   \n",
       "22  2020年04月02日          面议      深圳  本科及以上   1年以上   腾讯   \n",
       "23  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "24  2020年04月02日          面议      深圳  本科及以上   经验不限   腾讯   \n",
       "25  2020年04月02日          面议      深圳  本科及以上   经验不限   腾讯   \n",
       "26  2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "27  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "28  2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "29  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "30  2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "31  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "32  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "33  2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "34  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "35  2020年04月02日          面议      深圳  本科及以上   5年以上   腾讯   \n",
       "36  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "37  2020年04月02日          面议      深圳  本科及以上   2年以上   腾讯   \n",
       "38  2020年04月02日          面议      深圳  本科及以上   3年以上   腾讯   \n",
       "39  2020年04月02日          面议      北京  本科及以上   2年以上   腾讯   \n",
       "\n",
       "                                 职位  \\\n",
       "0                         腾讯云生态产品经理   \n",
       "1                          BI数据产品经理   \n",
       "2                              产品经理   \n",
       "3               CSIG16-垂类信息服务建设产品经理   \n",
       "4          30628-腾讯广告行业服务高级产品经理（北京）   \n",
       "5              31967-企鹅电竞用户增长渠道产品经理   \n",
       "6              32272-腾讯视频TV产品经理（深圳）   \n",
       "7               36960-数据产品经理（北京或深圳）   \n",
       "8                    30360-数据增长产品经理   \n",
       "9                  19332-产品运营经理（上海）   \n",
       "10              23674-腾讯新闻信息流体验产品经理   \n",
       "11         21309-在线教育--企鹅辅导AI产品策划经理   \n",
       "12              18427-理财通数据产品经理（深圳）   \n",
       "13                 30628-腾讯广告数据产品经理   \n",
       "14                  20052-搜索商业化产品经理   \n",
       "15         59577-29912-生态工具产品经理（北京）   \n",
       "16            36242-金融产品风险政策经理/高级经理   \n",
       "17            36242-金融产品风险管理经理/高级经理   \n",
       "18            36242-金融产品风险监控经理/高级经理   \n",
       "19                 34975-高级海外产品研究经理   \n",
       "20             TME-QQ音乐数据产品高级经理（深圳）   \n",
       "21           59545-30632-腾讯广告商业产品经理   \n",
       "22                     23518-社交产品经理   \n",
       "23                   25667-电销系统产品经理   \n",
       "24                22989-物联网生态平台产品经理   \n",
       "25              22989-智慧城市物联网中台产品经理   \n",
       "26           PCG22-创新项目数据中台产品经理（深圳）   \n",
       "27            25667-腾讯云合作产品交付经理（深圳）   \n",
       "28       59431-26699-智慧零售数据中心高级产品经理   \n",
       "29               28481-医疗行业产品经理（深圳）   \n",
       "30  59410-22989-腾讯云经管平台公共应用领域高级产品经理   \n",
       "31           27092-联运游戏商业策划产品经理（深圳）   \n",
       "32           30360-社交用户增长数据产品经理（深圳）   \n",
       "33            WXG04-小游戏平台策略产品经理（深圳）   \n",
       "34                22989-视频云产品经理（深圳）   \n",
       "35                  22989-CDN高级产品经理   \n",
       "36           22989-腾云扣钉产品经理(全资公司编制)   \n",
       "37   59381-22989-腾云扣钉产品运营经理(全资公司编制)   \n",
       "38               28606-微信支付财税行业产品经理   \n",
       "39                   20052-用户评估产品经理   \n",
       "\n",
       "                                           职位链接  \\\n",
       "0   https://www.liepin.com/job/1927237017.shtml   \n",
       "1   https://www.liepin.com/job/1926393923.shtml   \n",
       "2   https://www.liepin.com/job/1926042901.shtml   \n",
       "3   https://www.liepin.com/job/1927177593.shtml   \n",
       "4   https://www.liepin.com/job/1927177589.shtml   \n",
       "5   https://www.liepin.com/job/1927177563.shtml   \n",
       "6   https://www.liepin.com/job/1927177551.shtml   \n",
       "7   https://www.liepin.com/job/1927177539.shtml   \n",
       "8   https://www.liepin.com/job/1927177477.shtml   \n",
       "9   https://www.liepin.com/job/1927177475.shtml   \n",
       "10  https://www.liepin.com/job/1927177473.shtml   \n",
       "11  https://www.liepin.com/job/1927141655.shtml   \n",
       "12  https://www.liepin.com/job/1927141635.shtml   \n",
       "13  https://www.liepin.com/job/1927141599.shtml   \n",
       "14  https://www.liepin.com/job/1927141595.shtml   \n",
       "15  https://www.liepin.com/job/1927141575.shtml   \n",
       "16  https://www.liepin.com/job/1927130685.shtml   \n",
       "17  https://www.liepin.com/job/1927130683.shtml   \n",
       "18  https://www.liepin.com/job/1927130681.shtml   \n",
       "19  https://www.liepin.com/job/1927130311.shtml   \n",
       "20  https://www.liepin.com/job/1927100431.shtml   \n",
       "21  https://www.liepin.com/job/1927100425.shtml   \n",
       "22  https://www.liepin.com/job/1927100355.shtml   \n",
       "23  https://www.liepin.com/job/1927100333.shtml   \n",
       "24  https://www.liepin.com/job/1927038135.shtml   \n",
       "25  https://www.liepin.com/job/1927038133.shtml   \n",
       "26  https://www.liepin.com/job/1927038125.shtml   \n",
       "27  https://www.liepin.com/job/1927038111.shtml   \n",
       "28  https://www.liepin.com/job/1927038103.shtml   \n",
       "29  https://www.liepin.com/job/1927038063.shtml   \n",
       "30  https://www.liepin.com/job/1927038059.shtml   \n",
       "31  https://www.liepin.com/job/1927038041.shtml   \n",
       "32  https://www.liepin.com/job/1927038039.shtml   \n",
       "33  https://www.liepin.com/job/1927038037.shtml   \n",
       "34  https://www.liepin.com/job/1927038031.shtml   \n",
       "35  https://www.liepin.com/job/1927010835.shtml   \n",
       "36  https://www.liepin.com/job/1927010833.shtml   \n",
       "37  https://www.liepin.com/job/1927010831.shtml   \n",
       "38  https://www.liepin.com/job/1927010769.shtml   \n",
       "39  https://www.liepin.com/job/1927010763.shtml   \n",
       "\n",
       "                                       公司链接 热门公司类型  \n",
       "0   https://www.liepin.com/company/7983148/    好未来  \n",
       "1   https://www.liepin.com/company/7983148/    好未来  \n",
       "2   https://www.liepin.com/company/7983148/    好未来  \n",
       "3   https://www.liepin.com/company/7983148/    好未来  \n",
       "4   https://www.liepin.com/company/7983148/    好未来  \n",
       "5   https://www.liepin.com/company/7983148/    好未来  \n",
       "6   https://www.liepin.com/company/7983148/    好未来  \n",
       "7   https://www.liepin.com/company/7983148/    好未来  \n",
       "8   https://www.liepin.com/company/7983148/    好未来  \n",
       "9   https://www.liepin.com/company/7983148/    好未来  \n",
       "10  https://www.liepin.com/company/7983148/    好未来  \n",
       "11  https://www.liepin.com/company/7983148/    好未来  \n",
       "12  https://www.liepin.com/company/7983148/    好未来  \n",
       "13  https://www.liepin.com/company/7983148/    好未来  \n",
       "14  https://www.liepin.com/company/7983148/    好未来  \n",
       "15  https://www.liepin.com/company/7983148/    好未来  \n",
       "16  https://www.liepin.com/company/7983148/    好未来  \n",
       "17  https://www.liepin.com/company/7983148/    好未来  \n",
       "18  https://www.liepin.com/company/7983148/    好未来  \n",
       "19  https://www.liepin.com/company/7983148/    好未来  \n",
       "20  https://www.liepin.com/company/7983148/    好未来  \n",
       "21  https://www.liepin.com/company/7983148/    好未来  \n",
       "22  https://www.liepin.com/company/7983148/    好未来  \n",
       "23  https://www.liepin.com/company/7983148/    好未来  \n",
       "24  https://www.liepin.com/company/7983148/    好未来  \n",
       "25  https://www.liepin.com/company/7983148/    好未来  \n",
       "26  https://www.liepin.com/company/7983148/    好未来  \n",
       "27  https://www.liepin.com/company/7983148/    好未来  \n",
       "28  https://www.liepin.com/company/7983148/    好未来  \n",
       "29  https://www.liepin.com/company/7983148/    好未来  \n",
       "30  https://www.liepin.com/company/7983148/    好未来  \n",
       "31  https://www.liepin.com/company/7983148/    好未来  \n",
       "32  https://www.liepin.com/company/7983148/    好未来  \n",
       "33  https://www.liepin.com/company/7983148/    好未来  \n",
       "34  https://www.liepin.com/company/7983148/    好未来  \n",
       "35  https://www.liepin.com/company/7983148/    好未来  \n",
       "36  https://www.liepin.com/company/7983148/    好未来  \n",
       "37  https://www.liepin.com/company/7983148/    好未来  \n",
       "38  https://www.liepin.com/company/7983148/    好未来  \n",
       "39  https://www.liepin.com/company/7983148/    好未来  "
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "payload = 参数_配置url筛选['腾讯']\n",
    "list_cf = list()\n",
    "#keyword为用户体验，key为数据，职位囊括上市公司，中国500强的公司的爬取\n",
    "cf = requests( url, params = payload)#调用requests_liepin()函数\n",
    "cf = cf.assign(热门公司类型 = k)    \n",
    "list_cf.append(cf)\n",
    "    \n",
    "df_one_message = pd.concat(list_cf)  #对数据的合并与重塑\n",
    "df_one_message "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分页输出爬取的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "with pd.ExcelWriter('week03猎聘数据挖掘练习成果输出.xlsx' ,mode=\"w\") as fp:\n",
    "    df_all_message.to_excel(fp, sheet_name=\"交互设计师\")\n",
    "    数据挖掘.to_excel(fp, sheet_name=\"数据挖掘工程师\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "462.222px",
    "left": "54px",
    "top": "153.926px",
    "width": "284.433px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
